Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Merge branch 'master' into 196-disable-agents

Conflicts:
app/views/agents/diagram.html.erb
app/views/agents/show.html.erb

Manually Edited:
app/helpers/dot_helper.rb

Glenn 'devalias' Grant 11 years ago
parent
e71886a6f5 769771b38c
commit
03ceab8da5
55 changed files with 1036 additions and 491 deletions
Show Diff Stats
  1. 18 7
      .env.example
  2. 3 1
      .travis.yml
  3. 38 18
      Gemfile
  4. 108 94
      Gemfile.lock
  5. 1 1
      README.md
  6. 3 3
      app/assets/javascripts/application.js.coffee.erb
  7. 9 1
      app/controllers/agents_controller.rb
  8. 0 15
      app/helpers/application_helper.rb
  9. 48 0
      app/helpers/dot_helper.rb
  10. 16 3
      app/models/agent.rb
  11. 100 0
      app/models/agents/stubhub_agent.rb
  12. 20 3
      app/models/agents/trigger_agent.rb
  13. 10 10
      app/models/agents/twilio_agent.rb
  14. 92 35
      app/models/agents/website_agent.rb
  15. 2 2
      app/models/user.rb
  16. 1 11
      app/views/agents/diagram.html.erb
  17. 5 6
      app/views/agents/show.html.erb
  18. 1 1
      app/views/layouts/_messages.html.erb
  19. 3 0
      bin/bundle
  20. 4 0
      bin/rails
  21. 4 0
      bin/rake
  22. 1 13
      config/application.rb
  23. 7 8
      config/environments/development.rb
  24. 29 13
      config/environments/production.rb
  25. 0 76
      config/environments/staging.rb
  26. 6 4
      config/environments/test.rb
  27. 8 5
      config/initializers/devise.rb
  28. 1 1
      config/initializers/secret_token.rb
  29. 4 4
      config/routes.rb
  30. 1 1
      db/seeds.rb
  31. 3 0
      deployment/.chef/knife.rb
  32. 71 0
      deployment/Cheffile.lock
  33. 12 37
      deployment/Vagrantfile
  34. 1 0
      deployment/roles/huginn_development.json
  35. 1 1
      deployment/roles/huginn_production.json
  36. 1 1
      deployment/site-cookbooks/huginn_development/recipes/default.rb
  37. 0 58
      deployment/site-cookbooks/huginn_production/files/default/Gemfile
  38. 4 4
      deployment/site-cookbooks/huginn_production/files/default/Procfile
  39. 2 1
      deployment/site-cookbooks/huginn_production/files/default/env.example
  40. 5 6
      deployment/site-cookbooks/huginn_production/files/default/nginx.conf
  41. 4 2
      deployment/site-cookbooks/huginn_production/files/default/unicorn.rb
  42. 18 9
      deployment/site-cookbooks/huginn_production/recipes/default.rb
  43. 0 6
      deployment/solo.rb
  44. 2 2
      lib/rdbms_functions.rb
  45. 16 0
      spec/controllers/agents_controller_spec.rb
  46. 17 0
      spec/data_fixtures/stubhub_data.json
  47. 48 0
      spec/helpers/dot_helper_spec.rb
  48. 1 1
      spec/lib/utils_spec.rb
  49. 45 0
      spec/models/agent_spec.rb
  50. 1 1
      spec/models/agents/hipchat_agent_spec.rb
  51. 19 14
      spec/models/agents/public_transport_agent_spec.rb
  52. 67 0
      spec/models/agents/stubhub_agent_spec.rb
  53. 57 1
      spec/models/agents/trigger_agent_spec.rb
  54. 97 11
      spec/models/agents/website_agent_spec.rb
  55. 1 0
      spec/spec_helper.rb

+ 18 - 7
.env.example
View File 

@@ -31,6 +31,17 @@ DATABASE_PASSWORD=""
31 31 
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
32 32 
 # RAILS_ENV=production
33 33
34 
+# Should Rails force all requests to use SSL?
35 
+FORCE_SSL=false
36 
+
37 
+############################
38 
+#     Allowing Signups     #
39 
+############################
40 
+
41 
+# This invitation code will be required for users to signup with your Huginn installation.
42 
+# You can see its use in user.rb.  PLEASE CHANGE THIS!
43 
+INVITATION_CODE=try-huginn
44 
+
34 45 
 #############################
35 46 
 #    Email Configuration    #
36 47 
 #############################
 
@@ -52,13 +63,6 @@ SMTP_ENABLE_STARTTLS_AUTO=true
52 63 
 # The address from which system emails will appear to be sent.
53 64 
 EMAIL_FROM_ADDRESS=from_address@gmail.com
54 65
55 
-############################
56 
-#     Allowing Signups     #
57 
-############################
58 
-
59 
-# This invitation code will be required for users to signup with your Huginn installation.
60 
-# You can see its use in user.rb.
61 
-INVITATION_CODE=try-huginn
62 66
63 67 
 ###########################
64 68 
 #      Agent Logging      #
 
@@ -82,6 +86,13 @@ AWS_SANDBOX=false
82 86 
 #   Various Settings   #
83 87 
 ########################
84 88
89 
+# Specify the HTTP backend library for Faraday, used in WebsiteAgent.
90 
+# You can change this depending on the performance and stability you
91 
+# need for your service.  Any choice other than "typhoeus",
92 
+# "net_http", or "em_http" should require you to bundle a corresponding
93 
+# gem via Gemfile.
94 
+FARADAY_HTTP_BACKEND=typhoeus
95 
+
85 96 
 # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)]
86 97 
 # You should not allow this on a shared Huginn box because it is not secure.
87 98 
 ALLOW_JSONPATH_EVAL=false

+ 3 - 1
.travis.yml
View File 

@@ -1,5 +1,7 @@
1 1 
 language: ruby
2 2 
 bundler_args: --without development production
3 
+env:
4 
+  - APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d
3 5 
 rvm:
4 6 
   - 2.0.0
5 7 
   - 2.1.1
 
@@ -15,6 +17,6 @@ notifications:
15 17 
     channels:
16 18 
       - "chat.freenode.net#huginn"
17 19 
     template:
18 
-      - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}"
20 
+      - "<%{author}> %{branch} - %{commit} (%{commit_message}): %{message}"
19 21 
       - "Change view : %{compare_url}"
20 22 
       - "Build details : %{build_url}"

+ 38 - 18
Gemfile
View File 

@@ -1,14 +1,26 @@
1 1 
 source 'https://rubygems.org'
2 2
3 
-gem 'rails', '3.2.17'
4 
-gem 'mysql2', '~> 0.3.13'
5 
-gem 'devise', '~> 3.0.0'
6 
-gem 'kaminari', '~> 0.14.1'
3 
+gem 'protected_attributes', '~>1.0.7'
4 
+
5 
+gem 'rails', '4.1.0'
6 
+
7 
+case RUBY_PLATFORM
8 
+when /freebsd/i
9 
+  # Seems FreeBSD's zoneinfo is not exactly what tzinfo expects
10 
+  gem 'tzinfo-data'
11 
+else
12 
+  # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
13 
+  gem 'tzinfo-data', platforms: [:mswin]
14 
+end
15 
+
16 
+gem 'mysql2', '~> 0.3.15'
17 
+gem 'devise', '~> 3.2.4'
18 
+gem 'kaminari', '~> 0.15.1'
7 19 
 gem 'bootstrap-kaminari-views', '~> 0.0.2'
8 20 
 gem 'rufus-scheduler', '~> 3.0.7', require: false
9 
-gem 'json', '>= 1.7.7'
21 
+gem 'json', '~> 1.8.1'
10 22 
 gem 'jsonpath', '~> 0.5.3'
11 
-gem 'twilio-ruby', '~> 3.10.0'
23 
+gem 'twilio-ruby', '~> 3.11.5'
12 24 
 gem 'ruby-growl', '~> 4.1.0'
13 25
14 26 
 gem 'delayed_job', '~> 4.0.0'
 
@@ -20,27 +32,29 @@ gem 'daemons', '~> 1.1.9'
20 32
21 33 
 gem 'foreman', '~> 0.63.0'
22 34
23 
-gem 'sass-rails',   '~> 3.2.3'
24 
-gem 'coffee-rails', '~> 3.2.1'
25 
-gem 'uglifier', '>= 1.0.3'
26 
-gem 'select2-rails', '~> 3.4.3'
27 
-gem 'jquery-rails', '~> 3.0.4'
35 
+gem 'sass-rails',   '~> 4.0.0'
36 
+gem 'coffee-rails', '~> 4.0.0'
37 
+gem 'uglifier', '>= 1.3.0'
38 
+gem 'select2-rails', '~> 3.5.4'
39 
+gem 'jquery-rails', '~> 3.1.0'
28 40 
 gem 'ace-rails-ap', '~> 2.0.1'
29 41
30 42 
 # geokit-rails doesn't work with geokit 1.8.X but it specifies ~> 1.5
31 43 
 # in its own Gemfile.
32 
-gem 'geokit', '~> 1.6.7'
33 
-gem 'geokit-rails3', '~> 0.1.5'
44 
+gem 'geokit', '~> 1.8.4'
45 
+gem 'geokit-rails', '~> 2.0.1'
34 46
35 
-gem 'kramdown', '~> 1.1.0'
47 
+gem 'kramdown', '~> 1.3.3'
48 
+gem 'faraday', '~> 0.9.0'
49 
+gem 'faraday_middleware'
36 50 
 gem 'typhoeus', '~> 0.6.3'
37 
-gem 'nokogiri', '~> 1.6.0'
51 
+gem 'nokogiri', '~> 1.6.1'
38 52
39 
-gem 'wunderground', '~> 1.1.0'
53 
+gem 'wunderground', '~> 1.2.0'
40 54 
 gem 'forecast_io', '~> 2.0.0'
41 
-gem 'rturk', '~> 2.11.0'
55 
+gem 'rturk', '~> 2.12.1'
42 56
43 
-gem 'twitter', '~> 5.7.1'
57 
+gem 'twitter', '~> 5.8.0'
44 58 
 gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'master'
45 59 
 gem 'em-http-request', '~> 1.1.2'
46 60 
 gem 'weibo_2', '~> 0.1.4'
 
@@ -60,6 +74,12 @@ group :development, :test do
60 74 
   gem 'rspec'
61 75 
   gem 'shoulda-matchers'
62 76 
   gem 'rr'
77 
+  gem 'delorean'
63 78 
   gem 'webmock', require: false
64 79 
   gem 'coveralls', require: false
65 80 
 end
81 
+
82 
+group :production do
83 
+  gem 'dotenv-deployment'
84 
+  gem 'rack'
85 
+end

+ 108 - 94
Gemfile.lock
View File 

@@ -12,38 +12,35 @@ GEM
12 12 
   remote: https://rubygems.org/
13 13 
   specs:
14 14 
     ace-rails-ap (2.0.1)
15 
-    actionmailer (3.2.17)
16 
-      actionpack (= 3.2.17)
15 
+    actionmailer (4.1.0)
16 
+      actionpack (= 4.1.0)
17 
+      actionview (= 4.1.0)
17 18 
       mail (~> 2.5.4)
18 
-    actionpack (3.2.17)
19 
-      activemodel (= 3.2.17)
20 
-      activesupport (= 3.2.17)
21 
-      builder (~> 3.0.0)
19 
+    actionpack (4.1.0)
20 
+      actionview (= 4.1.0)
21 
+      activesupport (= 4.1.0)
22 
+      rack (~> 1.5.2)
23 
+      rack-test (~> 0.6.2)
24 
+    actionview (4.1.0)
25 
+      activesupport (= 4.1.0)
26 
+      builder (~> 3.1)
22 27 
       erubis (~> 2.7.0)
23 
-      journey (~> 1.0.4)
24 
-      rack (~> 1.4.5)
25 
-      rack-cache (~> 1.2)
26 
-      rack-test (~> 0.6.1)
27 
-      sprockets (~> 2.2.1)
28 
-    activemodel (3.2.17)
29 
-      activesupport (= 3.2.17)
30 
-      builder (~> 3.0.0)
31 
-    activerecord (3.2.17)
32 
-      activemodel (= 3.2.17)
33 
-      activesupport (= 3.2.17)
34 
-      arel (~> 3.0.2)
35 
-      tzinfo (~> 0.3.29)
36 
-    activeresource (3.2.17)
37 
-      activemodel (= 3.2.17)
38 
-      activesupport (= 3.2.17)
39 
-    activesupport (3.2.17)
40 
-      i18n (~> 0.6, >= 0.6.4)
41 
-      multi_json (~> 1.0)
28 
+    activemodel (4.1.0)
29 
+      activesupport (= 4.1.0)
30 
+      builder (~> 3.1)
31 
+    activerecord (4.1.0)
32 
+      activemodel (= 4.1.0)
33 
+      activesupport (= 4.1.0)
34 
+      arel (~> 5.0.0)
35 
+    activesupport (4.1.0)
36 
+      i18n (~> 0.6, >= 0.6.9)
37 
+      json (~> 1.7, >= 1.7.7)
38 
+      minitest (~> 5.1)
39 
+      thread_safe (~> 0.1)
40 
+      tzinfo (~> 1.1)
42 41 
     addressable (2.3.6)
43 
-    arel (3.0.3)
42 
+    arel (5.0.1.20140414130214)
44 43 
     bcrypt (3.1.7)
45 
-    bcrypt-ruby (3.1.5)
46 
-      bcrypt (>= 3.1.3)
47 44 
     better_errors (1.1.0)
48 45 
       coderay (>= 1.0.0)
49 46 
       erubis (>= 2.6.6)
 
@@ -53,11 +50,12 @@ GEM
53 50 
       kaminari (>= 0.13)
54 51 
       rails (>= 3.1)
55 52 
     buftok (0.2.0)
56 
-    builder (3.0.4)
53 
+    builder (3.2.2)
54 
+    chronic (0.10.2)
57 55 
     coderay (1.1.0)
58 
-    coffee-rails (3.2.2)
56 
+    coffee-rails (4.0.1)
59 57 
       coffee-script (>= 2.2.0)
60 
-      railties (~> 3.2.0)
58 
+      railties (>= 4.0.0, < 5.0)
61 59 
     coffee-script (2.2.0)
62 60 
       coffee-script-source
63 61 
       execjs
 
@@ -78,14 +76,18 @@ GEM
78 76 
     delayed_job_active_record (4.0.1)
79 77 
       activerecord (>= 3.0, < 4.2)
80 78 
       delayed_job (>= 3.0, < 4.1)
81 
-    devise (3.0.4)
82 
-      bcrypt-ruby (~> 3.0)
79 
+    delorean (2.1.0)
80 
+      chronic
81 
+    devise (3.2.4)
82 
+      bcrypt (~> 3.0)
83 83 
       orm_adapter (~> 0.1)
84 84 
       railties (>= 3.2.6, < 5)
85 
+      thread_safe (~> 0.1)
85 86 
       warden (~> 1.2.3)
86 87 
     diff-lcs (1.2.5)
87 88 
     docile (1.1.3)
88 89 
     dotenv (0.10.0)
90 
+    dotenv-deployment (0.0.2)
89 91 
     dotenv-rails (0.10.0)
90 92 
       dotenv (= 0.10.0)
91 93 
     em-http-request (1.1.2)
 
@@ -106,6 +108,8 @@ GEM
106 108 
     execjs (2.0.2)
107 109 
     faraday (0.9.0)
108 110 
       multipart-post (>= 1.2, < 3)
111 
+    faraday_middleware (0.9.1)
112 
+      faraday (>= 0.7.4, < 0.10)
109 113 
     ffi (1.9.3)
110 114 
     forecast_io (2.0.0)
111 115 
       faraday
 
@@ -114,11 +118,11 @@ GEM
114 118 
     foreman (0.63.0)
115 119 
       dotenv (>= 0.7)
116 120 
       thor (>= 0.13.6)
117 
-    geokit (1.6.7)
121 
+    geokit (1.8.4)
118 122 
       multi_json (>= 1.3.2)
119 
-    geokit-rails3 (0.1.5)
123 
+    geokit-rails (2.0.1)
120 124 
       geokit (~> 1.5)
121 
-      rails (~> 3.0)
125 
+      rails (>= 3.0)
122 126 
     hashie (2.0.5)
123 127 
     hike (1.2.3)
124 128 
     hipchat (1.1.0)
 
@@ -130,8 +134,7 @@ GEM
130 134 
       json (~> 1.8)
131 135 
       multi_xml (>= 0.5.2)
132 136 
     i18n (0.6.9)
133 
-    journey (1.0.4)
134 
-    jquery-rails (3.0.4)
137 
+    jquery-rails (3.1.0)
135 138 
       railties (>= 3.0, < 5.0)
136 139 
       thor (>= 0.14, < 2.0)
137 140 
     json (1.8.1)
 
@@ -139,10 +142,10 @@ GEM
139 142 
       multi_json
140 143 
     jwt (0.1.11)
141 144 
       multi_json (>= 1.5)
142 
-    kaminari (0.14.1)
145 
+    kaminari (0.15.1)
143 146 
       actionpack (>= 3.0.0)
144 147 
       activesupport (>= 3.0.0)
145 
-    kramdown (1.1.0)
148 
+    kramdown (1.3.3)
146 149 
     libv8 (3.16.14.3)
147 150 
     macaddr (1.7.1)
148 151 
       systemu (~> 2.6.2)
 
@@ -154,6 +157,7 @@ GEM
154 157 
     method_source (0.8.2)
155 158 
     mime-types (1.25.1)
156 159 
     mini_portile (0.5.3)
160 
+    minitest (5.3.3)
157 161 
     multi_json (1.9.2)
158 162 
     multi_xml (0.5.5)
159 163 
     multipart-post (2.0.0)
 
@@ -169,35 +173,31 @@ GEM
169 173 
       rack (~> 1.2)
170 174 
     orm_adapter (0.5.0)
171 175 
     polyglot (0.3.4)
176 
+    protected_attributes (1.0.7)
177 
+      activemodel (>= 4.0.1, < 5.0)
172 178 
     pry (0.9.12.6)
173 179 
       coderay (~> 1.0)
174 180 
       method_source (~> 0.8)
175 181 
       slop (~> 3.4)
176 
-    rack (1.4.5)
177 
-    rack-cache (1.2)
178 
-      rack (>= 0.4)
179 
-    rack-ssl (1.3.4)
180 
-      rack
182 
+    rack (1.5.2)
181 183 
     rack-test (0.6.2)
182 184 
       rack (>= 1.0)
183 
-    rails (3.2.17)
184 
-      actionmailer (= 3.2.17)
185 
-      actionpack (= 3.2.17)
186 
-      activerecord (= 3.2.17)
187 
-      activeresource (= 3.2.17)
188 
-      activesupport (= 3.2.17)
189 
-      bundler (~> 1.0)
190 
-      railties (= 3.2.17)
191 
-    railties (3.2.17)
192 
-      actionpack (= 3.2.17)
193 
-      activesupport (= 3.2.17)
194 
-      rack-ssl (~> 1.3.2)
185 
+    rails (4.1.0)
186 
+      actionmailer (= 4.1.0)
187 
+      actionpack (= 4.1.0)
188 
+      actionview (= 4.1.0)
189 
+      activemodel (= 4.1.0)
190 
+      activerecord (= 4.1.0)
191 
+      activesupport (= 4.1.0)
192 
+      bundler (>= 1.3.0, < 2.0)
193 
+      railties (= 4.1.0)
194 
+      sprockets-rails (~> 2.0)
195 
+    railties (4.1.0)
196 
+      actionpack (= 4.1.0)
197 
+      activesupport (= 4.1.0)
195 198 
       rake (>= 0.8.7)
196 
-      rdoc (~> 3.4)
197 
-      thor (>= 0.14.6, < 2.0)
198 
-    rake (10.2.2)
199 
-    rdoc (3.12.2)
200 
-      json (~> 1.4)
199 
+      thor (>= 0.18.1, < 2.0)
200 
+    rake (10.3.1)
201 201 
     ref (1.0.5)
202 202 
     rest-client (1.6.7)
203 203 
       mime-types (>= 1.16)
 
@@ -218,7 +218,7 @@ GEM
218 218 
       rspec-core (~> 2.14.0)
219 219 
       rspec-expectations (~> 2.14.0)
220 220 
       rspec-mocks (~> 2.14.0)
221 
-    rturk (2.11.3)
221 
+    rturk (2.12.1)
222 222 
       erector
223 223 
       nokogiri
224 224 
       rest-client
 
@@ -227,13 +227,13 @@ GEM
227 227 
     rufus-scheduler (3.0.7)
228 228 
       tzinfo
229 229 
     safe_yaml (1.0.2)
230 
-    sass (3.3.5)
231 
-    sass-rails (3.2.6)
232 
-      railties (~> 3.2.0)
233 
-      sass (>= 3.1.10)
234 
-      tilt (~> 1.3)
235 
-    select2-rails (3.4.9)
236 
-      sass-rails
230 
+    sass (3.2.19)
231 
+    sass-rails (4.0.3)
232 
+      railties (>= 4.0.0, < 5.0)
233 
+      sass (~> 3.2.0)
234 
+      sprockets (~> 2.8, <= 2.11.0)
235 
+      sprockets-rails (~> 2.0)
236 
+    select2-rails (3.5.4)
237 237 
       thor (~> 0.14)
238 238 
     shoulda-matchers (2.6.0)
239 239 
       activesupport (>= 3.0.0)
 
@@ -244,11 +244,15 @@ GEM
244 244 
       simplecov-html (~> 0.8.0)
245 245 
     simplecov-html (0.8.0)
246 246 
     slop (3.5.0)
247 
-    sprockets (2.2.2)
247 
+    sprockets (2.11.0)
248 248 
       hike (~> 1.2)
249 249 
       multi_json (~> 1.0)
250 250 
       rack (~> 1.0)
251 251 
       tilt (~> 1.1, != 1.3.0)
252 
+    sprockets-rails (2.1.3)
253 
+      actionpack (>= 3.0)
254 
+      activesupport (>= 3.0)
255 
+      sprockets (~> 2.8)
252 256 
     systemu (2.6.4)
253 257 
     term-ansicolor (1.3.0)
254 258 
       tins (~> 1.0)
 
@@ -262,11 +266,11 @@ GEM
262 266 
     treetop (1.4.15)
263 267 
       polyglot
264 268 
       polyglot (>= 0.3.1)
265 
-    twilio-ruby (3.10.1)
269 
+    twilio-ruby (3.11.5)
266 270 
       builder (>= 2.1.2)
267 271 
       jwt (>= 0.1.2)
268 272 
       multi_json (>= 1.3.0)
269 
-    twitter (5.7.1)
273 
+    twitter (5.8.0)
270 274 
       addressable (~> 2.3)
271 275 
       buftok (~> 0.2.0)
272 276 
       equalizer (~> 0.0.9)
 
@@ -279,7 +283,10 @@ GEM
279 283 
       simple_oauth (~> 0.2.0)
280 284 
     typhoeus (0.6.8)
281 285 
       ethon (>= 0.7.0)
282 
-    tzinfo (0.3.39)
286 
+    tzinfo (1.1.0)
287 
+      thread_safe (~> 0.1)
288 
+    tzinfo-data (1.2014.2)
289 
+      tzinfo (>= 1.0.0)
283 290 
     uglifier (2.5.0)
284 291 
       execjs (>= 0.3.0)
285 292 
       json (>= 1.8.0)
 
@@ -287,7 +294,7 @@ GEM
287 294 
       macaddr (~> 1.0)
288 295 
     warden (1.2.3)
289 296 
       rack (>= 1.0)
290 
-    webmock (1.13.0)
297 
+    webmock (1.17.4)
291 298 
       addressable (>= 2.2.7)
292 299 
       crack (>= 0.3.2)
293 300 
     weibo_2 (0.1.6)
 
@@ -295,7 +302,7 @@ GEM
295 302 
       multi_json (~> 1)
296 303 
       oauth2 (~> 0.9.1)
297 304 
       rest-client (~> 1.6.7)
298 
-    wunderground (1.1.0)
305 
+    wunderground (1.2.0)
299 306 
       addressable
300 307 
       httparty (> 0.6.0)
301 308 
       json (> 1.4.0)
 
@@ -308,43 +315,50 @@ DEPENDENCIES
308 315 
   better_errors
309 316 
   binding_of_caller
310 317 
   bootstrap-kaminari-views (~> 0.0.2)
311 
-  coffee-rails (~> 3.2.1)
318 
+  coffee-rails (~> 4.0.0)
312 319 
   coveralls
313 320 
   daemons (~> 1.1.9)
314 321 
   delayed_job (~> 4.0.0)
315 322 
   delayed_job_active_record (~> 4.0.0)
316 
-  devise (~> 3.0.0)
323 
+  delorean
324 
+  devise (~> 3.2.4)
325 
+  dotenv-deployment
317 326 
   dotenv-rails
318 327 
   em-http-request (~> 1.1.2)
328 
+  faraday (~> 0.9.0)
329 
+  faraday_middleware
319 330 
   forecast_io (~> 2.0.0)
320 331 
   foreman (~> 0.63.0)
321 
-  geokit (~> 1.6.7)
322 
-  geokit-rails3 (~> 0.1.5)
332 
+  geokit (~> 1.8.4)
333 
+  geokit-rails (~> 2.0.1)
323 334 
   hipchat (~> 1.1.0)
324 
-  jquery-rails (~> 3.0.4)
325 
-  json (>= 1.7.7)
335 
+  jquery-rails (~> 3.1.0)
336 
+  json (~> 1.8.1)
326 337 
   jsonpath (~> 0.5.3)
327 
-  kaminari (~> 0.14.1)
328 
-  kramdown (~> 1.1.0)
329 
-  mysql2 (~> 0.3.13)
330 
-  nokogiri (~> 1.6.0)
338 
+  kaminari (~> 0.15.1)
339 
+  kramdown (~> 1.3.3)
340 
+  mysql2 (~> 0.3.15)
341 
+  nokogiri (~> 1.6.1)
342 
+  protected_attributes (~> 1.0.7)
331 343 
   pry
332 
-  rails (= 3.2.17)
344 
+  rack
345 
+  rails (= 4.1.0)
333 346 
   rr
334 347 
   rspec
335 348 
   rspec-rails
336 
-  rturk (~> 2.11.0)
349 
+  rturk (~> 2.12.1)
337 350 
   ruby-growl (~> 4.1.0)
338 351 
   rufus-scheduler (~> 3.0.7)
339 
-  sass-rails (~> 3.2.3)
340 
-  select2-rails (~> 3.4.3)
352 
+  sass-rails (~> 4.0.0)
353 
+  select2-rails (~> 3.5.4)
341 354 
   shoulda-matchers
342 355 
   therubyracer (~> 0.12.1)
343 
-  twilio-ruby (~> 3.10.0)
344 
-  twitter (~> 5.7.1)
356 
+  twilio-ruby (~> 3.11.5)
357 
+  twitter (~> 5.8.0)
345 358 
   twitter-stream!
346 359 
   typhoeus (~> 0.6.3)
347 
-  uglifier (>= 1.0.3)
360 
+  tzinfo-data
361 
+  uglifier (>= 1.3.0)
348 362 
   webmock
349 363 
   weibo_2 (~> 0.1.4)
350 
-  wunderground (~> 1.1.0)
364 
+  wunderground (~> 1.2.0)

+ 1 - 1
README.md
View File 

@@ -104,5 +104,5 @@ Huginn is a work in progress and is hopefully just getting started.  Please get
104 104
105 105 
 Please fork, add specs, and send pull requests!
106 106
107 
-[![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.png)](https://coveralls.io/r/cantino/huginn) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
107 
+[![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.png)](https://coveralls.io/r/cantino/huginn) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge") [![Dependency Status](https://gemnasium.com/cantino/huginn.svg)](https://gemnasium.com/cantino/huginn)
108 108

+ 3 - 3
app/assets/javascripts/application.js.coffee.erb
View File 

@@ -56,9 +56,6 @@ $(document).ready ->
56 56 
   # JSON Editor
57 57 
   window.jsonEditor = setupJsonEditor()
58 58
59 
-  # Select2 Selects
60 
-  $(".select2").select2(width: 'resolve')
61 
-
62 59 
   # Flash
63 60 
   if $(".flash").length
64 61 
     setTimeout((-> $(".flash").slideUp(-> $(".flash").remove())), 5000)
 
@@ -155,6 +152,9 @@ $(document).ready ->
155 152
156 153 
   $("#agent_type").change() if $("#agent_type").length
157 154
155 
+  # Select2 Selects
156 
+  $(".select2").select2(width: 'resolve')
157 
+
158 158 
   if $(".schedule-region")
159 159 
     if $(".schedule-region").data("can-be-scheduled") == true
160 160 
       showSchedule()

+ 9 - 1
app/controllers/agents_controller.rb
View File 

@@ -1,4 +1,6 @@
1 1 
 class AgentsController < ApplicationController
2 
+  include DotHelper
3 
+
2 4 
   def index
3 5 
     @agents = current_user.agents.page(params[:page])
4 6 
 
@@ -101,7 +103,13 @@ class AgentsController < ApplicationController
101 103 
   end
102 104
103 105 
   def new
104 
-    @agent = current_user.agents.build
106 
+    agents = current_user.agents
107 
+
108 
+    if id = params[:id]
109 
+      @agent = agents.build_clone(agents.find(id))
110 
+    else
111 
+      @agent = agents.build
112 
+    end
105 113
106 114 
     respond_to do |format|
107 115 
       format.html

+ 0 - 15
app/helpers/application_helper.rb
View File 

@@ -16,19 +16,4 @@ module ApplicationHelper
16 16 
       link_to '<span class="label btn-danger">No</span>'.html_safe, agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details'))
17 17 
     end
18 18 
   end
19 
-
20 
-  def render_dot(dot_format_string)
21 
-    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
22 
-       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
23 
-          dot.print dot_format_string
24 
-          dot.close_write
25 
-          dot.read
26 
-        } rescue false)
27 
-      svg.html_safe
28 
-    else
29 
-      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
30 
-            uri.query = URI.encode_www_form(cht: 'gv', chl: dot_format_string)
31 
-          })
32 
-    end
33 
-  end
34 19 
 end

+ 48 - 0
app/helpers/dot_helper.rb
View File 

@@ -0,0 +1,48 @@
1 
+module DotHelper
2 
+  def render_agents_diagram(agents)
3 
+    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
4 
+       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
5 
+          dot.print agents_dot(agents, true)
6 
+          dot.close_write
7 
+          dot.read
8 
+        } rescue false)
9 
+      svg.html_safe
10 
+    else
11 
+      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
12 
+            uri.query = URI.encode_www_form(cht: 'gv', chl: agents_dot(agents))
13 
+          })
14 
+    end
15 
+  end
16 
+
17 
+  private
18 
+
19 
+  def dot_id(string)
20 
+    # Backslash escaping seems to work for the backslash itself,
21 
+    # despite the DOT language document.
22 
+    '"%s"' % string.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"")
23 
+  end
24 
+
25 
+  def agents_dot(agents, rich = false)
26 
+    "digraph foo {".tap { |dot|
27 
+      agents.each.with_index do |agent, index|
28 
+        if rich
29 
+          if agent.disabled
30 
+            dot << '%s[URL=%s] (Disabled);' % [dot_id(agent.name), dot_id(agent_path(agent.id))]
31 
+          else
32 
+            dot << '%s[URL=%s];' % [dot_id(agent.name), dot_id(agent_path(agent.id))]
33 
+          end
34 
+        else
35 
+          if agent.disabled
36 
+            dot << '%s (Disabled);' % dot_id(agent.name)
37 
+          else
38 
+            dot << '%s;' % dot_id(agent.name)
39 
+          end
40 
+        end
41 
+        agent.receivers.each do |receiver|
42 
+          dot << "%s->%s;" % [dot_id(agent.name), dot_id(receiver.name)]
43 
+        end
44 
+      end
45 
+      dot << "}"
46 
+    }
47 
+  end
48 
+end

+ 16 - 3
app/models/agent.rb
View File 

@@ -39,10 +39,10 @@ class Agent < ActiveRecord::Base
39 39 
   after_save :possibly_update_event_expirations
40 40
41 41 
   belongs_to :user, :inverse_of => :agents
42 
-  has_many :events, :dependent => :delete_all, :inverse_of => :agent, :order => "events.id desc"
42 
+  has_many :events, -> { order("events.id desc") }, :dependent => :delete_all, :inverse_of => :agent
43 43 
   has_one  :most_recent_event, :inverse_of => :agent, :class_name => "Event", :order => "events.id desc"
44 
-  has_many :logs, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog", :order => "agent_logs.id desc"
45 
-  has_many :received_events, :through => :sources, :class_name => "Event", :source => :events, :order => "events.id desc"
44 
+  has_many :logs,  -> { order("agent_logs.id desc") }, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog"
45 
+  has_many :received_events, -> { order("events.id desc") }, :through => :sources, :class_name => "Event", :source => :events
46 46 
   has_many :links_as_source, :dependent => :delete_all, :foreign_key => "source_id", :class_name => "Link", :inverse_of => :source
47 47 
   has_many :links_as_receiver, :dependent => :delete_all, :foreign_key => "receiver_id", :class_name => "Link", :inverse_of => :receiver
48 48 
   has_many :sources, :through => :links_as_receiver, :class_name => "Agent", :inverse_of => :receivers
 
@@ -230,6 +230,19 @@ class Agent < ActiveRecord::Base
230 230 
   # Class Methods
231 231
232 232 
   class << self
233 
+    def build_clone(original)
234 
+      new(original.slice(:type, :options, :schedule, :source_ids, :keep_events_for, :propagate_immediately)) { |clone|
235 
+        # Give it a unique name
236 
+        2.upto(count) do |i|
237 
+          name = '%s (%d)' % [original.name, i]
238 
+          unless exists?(name: name)
239 
+            clone.name = name
240 
+            break
241 
+          end
242 
+        end
243 
+      }
244 
+    end
245 
+
233 246 
     def cannot_be_scheduled!
234 247 
       @cannot_be_scheduled = true
235 248 
     end

+ 100 - 0
app/models/agents/stubhub_agent.rb
View File 

@@ -0,0 +1,100 @@
1 
+module Agents
2 
+  class StubhubAgent < Agent
3 
+    cannot_receive_events!
4 
+
5 
+    description <<-MD
6 
+      This StubHubAgent creates an event for a given StubHub Event. It can be used to track how many tickets are available for the event and the minimum and maximum price. All that is required is that you paste in the url from the actual event, e.g. http://www.stubhub.com/outside-lands-music-festival-tickets/outside-lands-music-festival-3-day-pass-san-francisco-golden-gate-park-polo-fields-8-8-2014-9020701/
7 
+    MD
8 
+
9 
+    event_description <<-MD
10 
+      Events looks like this:
11 
+        {
12 
+          "url": "http://stubhub.com/valid-event-url"
13 
+          "name": "Event Name"
14 
+          "date": "2014-08-01"
15 
+          "max_price": "999.99"
16 
+          "min_price": "100.99"
17 
+          "total_postings": "50"
18 
+          "total_tickets": "150"
19 
+          "venue_name": "Venue Name"
20 
+        }
21 
+    MD
22 
+
23 
+    default_schedule "every_1d"
24 
+
25 
+    def working?
26 
+      event_created_within?(1) && !recent_error_logs?
27 
+    end
28 
+
29 
+    def default_options
30 
+      { 'url' =>  'http://stubhub.com/enter-your-event-here' }
31 
+    end
32 
+
33 
+    def validate_options
34 
+      errors.add(:base, 'url is required') unless options['url'].present?
35 
+    end
36 
+
37 
+    def url
38 
+      options['url']
39 
+    end
40 
+
41 
+    def check
42 
+      create_event :payload => fetch_stubhub_data(url)
43 
+    end
44 
+
45 
+    def fetch_stubhub_data(url)
46 
+      StubhubFetcher.call(url)
47 
+    end
48 
+
49 
+    class StubhubFetcher
50 
+
51 
+      def self.call(url)
52 
+        new(url).fields
53 
+      end
54 
+
55 
+      def initialize(url)
56 
+        @url = url
57 
+      end
58 
+
59 
+      def event_id
60 
+        /(\d*)\/{0,1}\z/.match(url)[1]
61 
+      end
62 
+
63 
+      def base_url
64 
+       'http://www.stubhub.com/listingCatalog/select/?q='
65 
+      end
66 
+
67 
+      def build_url
68 
+        base_url + "%2B+stubhubDocumentType%3Aevent%0D%0A%2B+event_id%3A#{event_id}%0D%0A&start=0&rows=10&wt=json"
69 
+      end
70 
+
71 
+      def response
72 
+        uri = URI(build_url)
73 
+        Net::HTTP.get(uri)
74 
+      end
75 
+
76 
+      def parse_response
77 
+        JSON.parse(response)
78 
+      end
79 
+
80 
+      def fields
81 
+        stubhub_fields = parse_response['response']['docs'][0]
82 
+        {
83 
+          'url' => url,
84 
+          'name' => stubhub_fields['seo_description_en_US'],
85 
+          'date' => stubhub_fields['event_date_local'],
86 
+          'max_price' => stubhub_fields['maxPrice'].to_s,
87 
+          'min_price' => stubhub_fields['minPrice'].to_s,
88 
+          'total_postings' => stubhub_fields['totalPostings'].to_s,
89 
+          'total_tickets' => stubhub_fields['totalTickets'].to_i.to_s,
90 
+          'venue_name' => stubhub_fields['venue_name']
91 
+        }
92 
+      end
93 
+
94 
+      private
95 
+
96 
+      attr_reader :url
97 
+
98 
+    end
99 
+  end
100 
+end

+ 20 - 3
app/models/agents/trigger_agent.rb
View File 

@@ -15,6 +15,8 @@ module Agents
15 15
16 16 
       All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
17 17
18 
+      Set `keep_event` to `true` if you'd like to re-emit the incoming event, optionally merged with 'message' when provided.
19 
+
18 20 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
19 21 
     MD
20 22 
 
@@ -25,15 +27,20 @@ module Agents
25 27 
     MD
26 28
27 29 
     def validate_options
28 
-      unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['rules'].present? &&
30 
+      unless options['expected_receive_period_in_days'].present? && options['rules'].present? &&
29 31 
           options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? }
30 32 
         errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required")
31 33 
       end
34 
+
35 
+      errors.add(:base, "message is required unless 'keep_event' is 'true'") unless options['message'].present? || keep_event?
36 
+
37 
+      errors.add(:base, "keep_event, when present, must be 'true' or 'false'") unless options['keep_event'].blank? || %w[true false].include?(options['keep_event'])
32 38 
     end
33 39
34 40 
     def default_options
35 41 
       {
36 42 
         'expected_receive_period_in_days' => "2",
43 
+        'keep_event' => 'false',
37 44 
         'rules' => [{
38 45 
                       'type' => "regex",
39 46 
                       'value' => "foo\\d+bar",
 
@@ -79,10 +86,20 @@ module Agents
79 86 
         end
80 87
81 88 
         if match
82 
-          create_event :payload => { 'message' => make_message(event[:payload]) } # Maybe this should include the
83 
-                                                                                  # original event as well?
89 
+          if keep_event?
90 
+            payload = event.payload.dup
91 
+            payload['message'] = make_message(event[:payload]) if options['message'].present?
92 
+          else
93 
+            payload = { 'message' => make_message(event[:payload]) }
94 
+          end
95 
+
96 
+          create_event :payload => payload
84 97 
         end
85 98 
       end
86 99 
     end
100 
+
101 
+    def keep_event?
102 
+      options['keep_event'] == 'true'
103 
+    end
87 104 
   end
88 105 
 end

+ 10 - 10
app/models/agents/twilio_agent.rb
View File 

@@ -7,17 +7,16 @@ module Agents
7 7 
     cannot_create_events!
8 8
9 9 
     description <<-MD
10 
-      The TwilioAgent receives and collects events and sends them via text message or gives you a call when scheduled.
10 
+      The TwilioAgent receives and collects events and sends them via text message (up to 160 characters) or gives you a call when scheduled.
11 11
12 
-      It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
12 
+      It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use the EventFormattingAgent if your event does not provide these keys.
13 13
14 14 
       Set `receiver_cell` to the number to receive text messages/call and `sender_cell` to the number sending them.
15 15
16 16 
       `expected_receive_period_in_days` is maximum number of days that you would expect to pass between events being received by this agent.
17 17
18 
-      If you would like to receive calls, then set `receive_call` to true. `server_url` needs to be
19 
-      filled only if you are making calls. Dont forget to include http/https in `server_url`.
20 
-
18 
+      If you would like to receive calls, set `receive_call` to `true`. In this case, `server_url` must be set to the URL of your
19 
+      Huginn installation (probably "https://#{ENV['DOMAIN']}"), which must be web-accessible.  Be sure to set http/https correctly.
21 20 
     MD
22 21
23 22 
     def default_options
 
@@ -43,13 +42,14 @@ module Agents
43 42 
       @client = Twilio::REST::Client.new options['account_sid'], options['auth_token']
44 43 
       memory['pending_calls'] ||= {}
45 44 
       incoming_events.each do |event|
46 
-        message = (event.payload['message'] || event.payload['text'] || event.payload['sms']).to_s
47 
-        if message != ""
45 
+        message = (event.payload['message'].presence || event.payload['text'].presence || event.payload['sms'].presence).to_s
46 
+        if message.present?
48 47 
           if options['receive_call'].to_s == 'true'
49 48 
             secret = SecureRandom.hex 3
50 49 
             memory['pending_calls'][secret] = message
51 50 
             make_call secret
52 51 
           end
52 
+
53 53 
           if options['receive_text'].to_s == 'true'
54 54 
             message = message.slice 0..160
55 55 
             send_message message
 
@@ -71,11 +71,11 @@ module Agents
71 71 
     def make_call(secret)
72 72 
       @client.account.calls.create :from => options['sender_cell'],
73 73 
                                    :to => options['receiver_cell'],
74 
-                                   :url => post_url(options['server_url'],secret)
74 
+                                   :url => post_url(options['server_url'], secret)
75 75 
     end
76 76
77 
-    def post_url(server_url,secret)
78 
-      "#{server_url}/users/#{self.user.id}/web_requests/#{self.id}/#{secret}"
77 
+    def post_url(server_url, secret)
78 
+      "#{server_url}/users/#{user.id}/web_requests/#{id}/#{secret}"
79 79 
     end
80 80
81 81 
     def receive_web_request(params, method, format)

+ 92 - 35
app/models/agents/website_agent.rb
View File 

@@ -1,10 +1,10 @@
1 1 
 require 'nokogiri'
2 
-require 'typhoeus'
2 
+require 'faraday'
3 
+require 'faraday_middleware'
3 4 
 require 'date'
4 5
5 6 
 module Agents
6 7 
   class WebsiteAgent < Agent
7 
-    cannot_receive_events!
8 8
9 9 
     default_schedule "every_12h"
10 10 
 
@@ -22,30 +22,36 @@ module Agents
22 22
23 23 
       To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
24 24
25 
-      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:
25 
+      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `"text": true` or `attr` pointing to an attribute name to grab.  An example:
26 26
27 
-          'extract': {
28 
-            'url': { 'css': "#comic img", 'attr': "src" },
29 
-            'title': { 'css': "#comic img", 'attr': "title" },
30 
-            'body_text': { 'css': "div.main", 'text': true }
27 
+          "extract": {
28 
+            "url": { "css": "#comic img", "attr": "src" },
29 
+            "title": { "css": "#comic img", "attr": "title" },
30 
+            "body_text": { "css": "div.main", "text": true }
31 31 
           }
32 32
33 33 
       When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:
34 34
35 
-          'extract': {
36 
-            'title': { 'path': "results.data[*].title" },
37 
-            'description': { 'path': "results.data[*].description" }
35 
+          "extract": {
36 
+            "title": { "path": "results.data[*].title" },
37 
+            "description": { "path": "results.data[*].description" }
38 38 
           }
39 39
40 40 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
41 41
42 
-      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.
42 
+      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`.
43 43
44 44 
       Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.  This is only used to set the "working" status.
45 45
46 46 
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
47 47
48 48 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
49 
+
50 
+      Set `user_agent` to a custom User-Agent name if the website does not like the default value ("Faraday v#{Faraday::VERSION}").
51 
+
52 
+      The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
53 
+
54 
+      The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
49 55 
     MD
50 56
51 57 
     event_description do
 
@@ -102,30 +108,33 @@ module Agents
102 108 
           errors.add(:base, "force_encoding must be a string")
103 109 
         end
104 110 
       end
105 
-    end
106 111
107 
-    def check
108 
-      hydra = Typhoeus::Hydra.new
109 
-      log "Fetching #{options['url']}"
110 
-      request_opts = { :followlocation => true }
111 
-      request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
112 
+      if options['user_agent'].present?
113 
+        errors.add(:base, "user_agent must be a string") unless options['user_agent'].is_a?(String)
114 
+      end
112 115
113 
-      requests = []
116 
+      unless headers.is_a?(Hash)
117 
+        errors.add(:base, "if provided, headers must be a hash")
118 
+      end
114 119
115 
-      if options['url'].kind_of?(Array)
116 
-        options['url'].each do |url|
117 
-           requests.push(Typhoeus::Request.new(url, request_opts))
118 
-        end
119 
-      else
120 
-        requests.push(Typhoeus::Request.new(options['url'], request_opts))
120 
+      begin
121 
+        basic_auth_credentials()
122 
+      rescue => e
123 
+        errors.add(:base, e.message)
121 124 
       end
125 
+    end
122 126
123 
-      requests.each do |request|
124 
-        request.on_failure do |response|
125 
-          error "Failed: #{response.inspect}"
126 
-        end
127 
+    def check
128 
+      check_url options['url']
129 
+    end
127 130
128 
-        request.on_success do |response|
131 
+    def check_url(in_url)
132 
+      return unless in_url.present?
133 
+
134 
+      Array(in_url).each do |url|
135 
+        log "Fetching #{url}"
136 
+        response = faraday.get(url)
137 
+        if response.success?
129 138 
           body = response.body
130 139 
           if (encoding = options['force_encoding']).present?
131 140 
             body = body.encode(Encoding::UTF_8, encoding)
 
@@ -150,7 +159,7 @@ module Agents
150 159 
                 when xpath = extraction_details['xpath']
151 160 
                   nodes = doc.xpath(xpath)
152 161 
                 else
153 
-                  error "'css' or 'xpath' is required for HTML or XML extraction"
162 
+                  error '"css" or "xpath" is required for HTML or XML extraction'
154 163 
                   return
155 164 
                 end
156 165 
                 unless Nokogiri::XML::NodeSet === nodes
 
@@ -163,7 +172,7 @@ module Agents
163 172 
                   elsif extraction_details['text']
164 173 
                     node.text()
165 174 
                   else
166 
-                    error "'attr' or 'text' is required on HTML or XML extraction patterns"
175 
+                    error '"attr" or "text" is required on HTML or XML extraction patterns'
167 176 
                     return
168 177 
                   end
169 178 
                 }
 
@@ -178,14 +187,14 @@ module Agents
178 187 
               error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}"
179 188 
               return
180 189 
             end
181 
-
190 
+
182 191 
             old_events = previous_payloads num_unique_lengths.first
183 192 
             num_unique_lengths.first.times do |index|
184 193 
               result = {}
185 194 
               options['extract'].keys.each do |name|
186 195 
                 result[name] = output[name][index]
187 196 
                 if name.to_s == 'url'
188 
-                  result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
197 
+                  result[name] = (response.env[:url] + result[name]).to_s
189 198 
                 end
190 199 
               end
191 200 
 
@@ -195,10 +204,16 @@ module Agents
195 204 
               end
196 205 
             end
197 206 
           end
207 
+        else
208 
+          error "Failed: #{response.inspect}"
198 209 
         end
210 
+      end
211 
+    end
199 212
200 
-        hydra.queue request
201 
-        hydra.run
213 
+    def receive(incoming_events)
214 
+      incoming_events.each do |event|
215 
+        url_to_scrape = event.payload['url']
216 
+        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
202 217 
       end
203 218 
     end
204 219 
 
@@ -275,5 +290,47 @@ module Agents
275 290 
         false
276 291 
       end
277 292 
     end
293 
+
294 
+    def faraday
295 
+      @faraday ||= Faraday.new { |builder|
296 
+        builder.headers = headers if headers.length > 0
297 
+
298 
+        if (user_agent = options['user_agent']).present?
299 
+          builder.headers[:user_agent] = user_agent
300 
+        end
301 
+
302 
+        builder.use FaradayMiddleware::FollowRedirects
303 
+        builder.request :url_encoded
304 
+        if userinfo = basic_auth_credentials()
305 
+          builder.request :basic_auth, *userinfo
306 
+        end
307 
+
308 
+        case backend = faraday_backend
309 
+        when :typhoeus
310 
+          require 'typhoeus/adapters/faraday'
311 
+        end
312 
+        builder.adapter backend
313 
+      }
314 
+    end
315 
+
316 
+    def faraday_backend
317 
+      ENV.fetch('FARADAY_HTTP_BACKEND', 'typhoeus').to_sym
318 
+    end
319 
+
320 
+    def basic_auth_credentials
321 
+      case value = options['basic_auth']
322 
+      when nil, ''
323 
+        return nil
324 
+      when Array
325 
+        return value if value.size == 2
326 
+      when /:/
327 
+        return value.split(/:/, 2)
328 
+      end
329 
+      raise "bad value for basic_auth: #{value.inspect}"
330 
+    end
331 
+
332 
+    def headers
333 
+      options['headers'].presence || {}
334 
+    end
278 335 
   end
279 336 
 end

+ 2 - 2
app/models/user.rb
View File 

@@ -23,8 +23,8 @@ class User < ActiveRecord::Base
23 23 
   validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid"
24 24
25 25 
   has_many :user_credentials, :dependent => :destroy, :inverse_of => :user
26 
-  has_many :events, :order => "events.created_at desc", :dependent => :delete_all, :inverse_of => :user
27 
-  has_many :agents, :order => "agents.created_at desc", :dependent => :destroy, :inverse_of => :user
26 
+  has_many :events, -> { order("events.created_at desc") }, :dependent => :delete_all, :inverse_of => :user
27 
+  has_many :agents, -> { order("agents.created_at desc") }, :dependent => :destroy, :inverse_of => :user
28 28 
   has_many :logs, :through => :agents, :class_name => "AgentLog"
29 29
30 30 
   # Allow users to login via either email or username.

+ 1 - 11
app/views/agents/diagram.html.erb
View File 

@@ -9,17 +9,7 @@
9 9 
       </div>
10 10
11 11 
       <div class='digraph'>
12 
-        <%
13 
-          dot_format_string = "digraph foo {"
14 
-          @agents.each.with_index do |agent, index|
15 
-            dot_format_string += "\"#{agent.name}#{" (Disabled)" if agent.disabled?}\";"
16 
-            agent.receivers.each do |receiver|
17 
-              dot_format_string += "\"#{agent.name}#{" (Disabled)" if agent.disabled?}\"->\"#{receiver.name}#{" (Disabled)" if receiver.disabled?}\";"
18 
-            end
19 
-           end
20 
-           dot_format_string = dot_format_string + "}"
21 
-        %>
22 
-        <%= render_dot(dot_format_string) %>
12 
+        <%= render_agents_diagram(@agents) %>
23 13 
       </div>
24 14 
     </div>
25 15 
   </div>

+ 5 - 6
app/views/agents/show.html.erb
View File 

@@ -14,12 +14,10 @@
14 14 
           <% end %>
15 15 
           <li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>" class='<%= @agent.recent_error_logs? ? 'recent-errors' : '' %>'><i class='icon-list-alt'></i> Logs</a></li>
16 16
17 
-          <% if @agent.can_create_events? %>
18 
-            <% if @agent.events.count > 0 %>
19 
-              <li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
20 
-            <% else %>
21 
-              <li class='disabled'><a><i class='icon-random'></i> Events</a></li>
22 
-            <% end %>
17 
+          <% if @agent.can_create_events? && @agent.events.count > 0 %>
18 
+            <li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
19 
+          <% else %>
20 
+            <li class='disabled'><a><i class='icon-random'></i> Events</a></li>
23 21 
           <% end %>
24 22
25 23 
           <li class="dropdown">
 
@@ -32,6 +30,7 @@
32 30 
               <% end %>
33 31
34 32 
               <li><%= link_to '<i class="icon-pencil"></i> Edit'.html_safe, edit_agent_path(@agent) %></li>
33 
+              <li><%= link_to '<i class="icon-plus"></i> Clone'.html_safe, new_agent_path(id: @agent) %></li>
35 34
36 35 
               <li>
37 36 
                 <% if !@agent.disabled? %>

+ 1 - 1
app/views/layouts/_messages.html.erb
View File 

@@ -1,7 +1,7 @@
1 1 
 <% if flash.keys.length > 0 %>
2 2 
   <div class="flash">
3 3 
     <% flash.each do |name, msg| %>
4 
-      <div class="alert alert-<%= name == :notice ? "success" : "error" %>">
4 
+      <div class="alert alert-<%= name.to_sym == :notice ? "success" : "error" %>">
5 5 
         <a class="close" data-dismiss="alert">&#215;</a>
6 6 
         <%= content_tag :div, msg, :id => "flash_#{name}" if msg.is_a?(String) %>
7 7 
       </div>

+ 3 - 0
bin/bundle
View File 

@@ -0,0 +1,3 @@
1 
+#!/usr/bin/env ruby
2 
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
3 
+load Gem.bin_path('bundler', 'bundle')

+ 4 - 0
bin/rails
View File 

@@ -0,0 +1,4 @@
1 
+#!/usr/bin/env ruby
2 
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
3 
+require_relative '../config/boot'
4 
+require 'rails/commands'

+ 4 - 0
bin/rake
View File 

@@ -0,0 +1,4 @@
1 
+#!/usr/bin/env ruby
2 
+require_relative '../config/boot'
3 
+require 'rake'
4 
+Rake.application.run

+ 1 - 13
config/application.rb
View File 

@@ -2,12 +2,7 @@ require File.expand_path('../boot', __FILE__)
2 2
3 3 
 require 'rails/all'
4 4
5 
-if defined?(Bundler)
6 
-  # If you precompile assets before deploying to production, use this line
7 
-  Bundler.require(*Rails.groups(:assets => %w(development test)))
8 
-  # If you want your assets lazily compiled in production, use this line
9 
-  # Bundler.require(:default, :assets, Rails.env)
10 
-end
5 
+Bundler.require(:default, Rails.env)
11 6
12 7 
 module Huginn
13 8 
   class Application < Rails::Application
 
@@ -18,10 +13,6 @@ module Huginn
18 13 
     # Custom directories with classes and modules you want to be autoloadable.
19 14 
     config.autoload_paths += %W(#{config.root}/lib)
20 15
21 
-    # Only load the plugins named here, in the order given (default is alphabetical).
22 
-    # :all can be used as a placeholder for all plugins not explicitly named.
23 
-    # config.plugins = [ :exception_notification, :ssl_requirement, :all ]
24 
-
25 16 
     # Activate observers that should always be running.
26 17 
     # config.active_record.observers = :cacher, :garbage_collector, :forum_observer
27 18 
 
@@ -56,8 +47,5 @@ module Huginn
56 47 
     # Enable the asset pipeline
57 48 
     config.assets.enabled = true
58 49 
     config.assets.initialize_on_precompile = false
59 
-
60 
-    # Version of your assets, change this if you want to expire all your assets
61 
-    config.assets.version = '1.0'
62 50 
   end
63 51 
 end

+ 7 - 8
config/environments/development.rb
View File 

@@ -8,8 +8,11 @@ Huginn::Application.configure do
8 8 
   # since you don't have to restart the web server when you make code changes.
9 9 
   config.cache_classes = false
10 10
11 
-  # Log error messages when you accidentally call methods on nil.
12 
-  config.whiny_nils = true
11 
+  # Eager load code on boot. This eager loads most of Rails and
12 
+  # your application in memory, allowing both threaded web servers
13 
+  # and those relying on copy on write to perform better.
14 
+  # Rake tasks automatically ignore this option for performance.
15 
+  config.eager_load = false
13 16
14 17 
   # Show full error reports and disable caching
15 18 
   config.consider_all_requests_local       = true
 
@@ -24,12 +27,8 @@ Huginn::Application.configure do
24 27 
   # Raise exception on mass assignment protection for Active Record models
25 28 
   config.active_record.mass_assignment_sanitizer = :strict
26 29
27 
-  # Log the query plan for queries taking more than this (works
28 
-  # with SQLite, MySQL, and PostgreSQL)
29 
-  config.active_record.auto_explain_threshold_in_seconds = 0.5
30 
-
31 
-  # Do not compress assets
32 
-  config.assets.compress = false
30 
+  # Raise an error on page load if there are pending migrations.
31 
+  config.active_record.migration_error = :page_load
33 32
34 33 
   # Expands the lines which load the assets
35 34 
   config.assets.debug = true

+ 29 - 13
config/environments/production.rb
View File 

@@ -4,31 +4,41 @@ Huginn::Application.configure do
4 4 
   # Code is not reloaded between requests
5 5 
   config.cache_classes = true
6 6
7 
+  # Eager load code on boot. This eager loads most of Rails and
8 
+  # your application in memory, allowing both threaded web servers
9 
+  # and those relying on copy on write to perform better.
10 
+  # Rake tasks automatically ignore this option for performance.
11 
+  config.eager_load = true
12 
+
7 13 
   # Full error reports are disabled and caching is turned on
8 14 
   config.consider_all_requests_local       = false
9 15 
   config.action_controller.perform_caching = true
10 16
17 
+  # Enable Rack::Cache to put a simple HTTP cache in front of your application
18 
+  # Add `rack-cache` to your Gemfile before enabling this.
19 
+  # For large-scale production use, consider using a caching reverse proxy like nginx, varnish or squid.
20 
+  # config.action_dispatch.rack_cache = true
21 
+
11 22 
   # Disable Rails's static asset server (Apache or nginx will already do this)
12 23 
   config.serve_static_assets = false
13 24
14 25 
   # Compress JavaScripts and CSS
15 
-  config.assets.compress = true
26 
+  config.assets.js_compressor  = :uglifier
27 
+  config.assets.css_compressor = :sass
16 28
17 29 
   # Don't fallback to assets pipeline if a precompiled asset is missed
18 30 
   config.assets.compile = false
19 31
20 32 
   # Generate digests for assets URLs
21 33 
   config.assets.digest = true
34 
+  config.assets.precompile += %w(*.png *.jpg *.jpeg *.gif)
22 35
23 
-  # Defaults to nil and saved in location specified by config.assets.prefix
24 
-  # config.assets.manifest = YOUR_PATH
25 
-
26 
-  # Specifies the header that your server uses for sending files
36 
+  # Specifies the header that your server uses for sending files.
27 37 
   # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
28 38 
   # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
29 39
30 40 
   # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
31 
-  config.force_ssl = true
41 
+  config.force_ssl = ENV['FORCE_SSL'].present? && ENV['FORCE_SSL'] == 'true' ? true : false
32 42
33 43 
   # See everything in the log (default is :info)
34 44 
   # config.log_level = :debug
 
@@ -50,19 +60,25 @@ Huginn::Application.configure do
50 60 
   # Precompile additional assets (application.js.coffee.erb, application.css, and all non-JS/CSS are already added)
51 61 
   config.assets.precompile += %w( graphing.js user_credentials.js )
52 62
53 
-  # Enable threaded mode
54 
-  # config.threadsafe!
63 
+  # Ignore bad email addresses and do not raise email delivery errors.
64 
+  # Set this to true and configure the email server for immediate delivery to raise delivery errors.
65 
+  # config.action_mailer.raise_delivery_errors = false
55 66
56 67 
   # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
57 
-  # the I18n.default_locale when a translation can not be found)
68 
+  # the I18n.default_locale when a translation cannot be found).
58 69 
   config.i18n.fallbacks = true
59 70
60 71 
   # Send deprecation notices to registered listeners
61 72 
   config.active_support.deprecation = :notify
62 73
63 
-  # Log the query plan for queries taking more than this (works
64 
-  # with SQLite, MySQL, and PostgreSQL)
65 
-  # config.active_record.auto_explain_threshold_in_seconds = 0.5
74 
+  # Disable automatic flushing of the log to improve performance.
75 
+  # config.autoflush_log = false
76 
+
77 
+  # Use default logging formatter so that PID and timestamp are not suppressed.
78 
+  config.log_formatter = ::Logger::Formatter.new
79 
+
80 
+  # Do not dump schema after migrations.
81 
+  config.active_record.dump_schema_after_migration = false
66 82
67 83 
   config.action_mailer.default_url_options = { :host => ENV['DOMAIN'] }
68 84 
   config.action_mailer.asset_host = ENV['DOMAIN']
 
@@ -73,4 +89,4 @@ Huginn::Application.configure do
73 89 
   config.action_mailer.raise_delivery_errors = true
74 90 
   config.action_mailer.delivery_method = :smtp
75 91 
   # smtp_settings moved to config/initializers/action_mailer.rb
76 
-end
92 
+end

+ 0 - 76
config/environments/staging.rb
View File 

@@ -1,76 +0,0 @@
1 
-Huginn::Application.configure do
2 
-  # Settings specified here will take precedence over those in config/application.rb
3 
-
4 
-  # Code is not reloaded between requests
5 
-  config.cache_classes = true
6 
-
7 
-  # Full error reports are disabled and caching is turned on
8 
-  config.consider_all_requests_local       = false
9 
-  config.action_controller.perform_caching = true
10 
-
11 
-  # Disable Rails's static asset server (Apache or nginx will already do this)
12 
-  config.serve_static_assets = false
13 
-
14 
-  # Compress JavaScripts and CSS
15 
-  config.assets.compress = true
16 
-
17 
-  # Don't fallback to assets pipeline if a precompiled asset is missed
18 
-  config.assets.compile = false
19 
-
20 
-  # Generate digests for assets URLs
21 
-  config.assets.digest = true
22 
-
23 
-  # Defaults to nil and saved in location specified by config.assets.prefix
24 
-  # config.assets.manifest = YOUR_PATH
25 
-
26 
-  # Specifies the header that your server uses for sending files
27 
-  # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
28 
-  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
29 
-
30 
-  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
31 
-  config.force_ssl = true
32 
-
33 
-  # See everything in the log (default is :info)
34 
-  # config.log_level = :debug
35 
-
36 
-  # Prepend all log lines with the following tags
37 
-  config.log_tags = [ :uuid ] # :subdomain
38 
-
39 
-  # Use a different logger for distributed setups
40 
-  # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
41 
-
42 
-  # Use a different cache store in production
43 
-  # config.cache_store = :mem_cache_store
44 
-
45 
-  # Enable serving of images, stylesheets, and JavaScripts from an asset server
46 
-  if ENV['ASSET_HOST'].present?
47 
-    config.action_controller.asset_host = ENV['ASSET_HOST']
48 
-  end
49 
-
50 
-  # Precompile additional assets (application.js.coffee.erb, application.css, and all non-JS/CSS are already added)
51 
-  config.assets.precompile += %w( graphing.js )
52 
-
53 
-  # Enable threaded mode
54 
-  # config.threadsafe!
55 
-
56 
-  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
57 
-  # the I18n.default_locale when a translation can not be found)
58 
-  config.i18n.fallbacks = true
59 
-
60 
-  # Send deprecation notices to registered listeners
61 
-  config.active_support.deprecation = :notify
62 
-
63 
-  # Log the query plan for queries taking more than this (works
64 
-  # with SQLite, MySQL, and PostgreSQL)
65 
-  # config.active_record.auto_explain_threshold_in_seconds = 0.5
66 
-
67 
-  config.action_mailer.default_url_options = { :host => ENV['DOMAIN'] }
68 
-  config.action_mailer.asset_host = ENV['DOMAIN']
69 
-  if ENV['ASSET_HOST']
70 
-    config.action_mailer.asset_host = ENV['ASSET_HOST']
71 
-  end
72 
-  config.action_mailer.perform_deliveries = true
73 
-  config.action_mailer.raise_delivery_errors = true
74 
-  config.action_mailer.delivery_method = :smtp
75 
-  # smtp_settings moved to config/initializers/action_mailer.rb
76 
-end

+ 6 - 4
config/environments/test.rb
View File 

@@ -7,13 +7,15 @@ Huginn::Application.configure do
7 7 
   # and recreated between test runs. Don't rely on the data there!
8 8 
   config.cache_classes = true
9 9
10 
+  # Do not eager load code on boot. This avoids loading your whole application
11 
+  # just for the purpose of running a single test. If you are using a tool that
12 
+  # preloads Rails for running tests, you may have to set it to true.
13 
+  config.eager_load = false
14 
+
10 15 
   # Configure static asset server for tests with Cache-Control for performance
11 16 
   config.serve_static_assets = true
12 17 
   config.static_cache_control = "public, max-age=3600"
13 18
14 
-  # Log error messages when you accidentally call methods on nil
15 
-  config.whiny_nils = true
16 
-
17 19 
   # Show full error reports and disable caching
18 20 
   config.consider_all_requests_local       = true
19 21 
   config.action_controller.perform_caching = false
 
@@ -22,7 +24,7 @@ Huginn::Application.configure do
22 24 
   config.action_dispatch.show_exceptions = false
23 25
24 26 
   # Disable request forgery protection in test environment
25 
-  config.action_controller.allow_forgery_protection    = false
27 
+  config.action_controller.allow_forgery_protection = false
26 28
27 29 
   # Tell Action Mailer not to deliver emails to the real world.
28 30 
   # The :test delivery method accumulates sent emails in the

+ 8 - 5
config/initializers/devise.rb
View File 

@@ -3,7 +3,8 @@
3 3 
 Devise.setup do |config|
4 4 
   # ==> Mailer Configuration
5 5 
   # Configure the e-mail address which will be shown in Devise::Mailer,
6 
-  # note that it will be overwritten if you use your own mailer class with default "from" parameter.
6 
+  # note that it will be overwritten if you use your own mailer class
7 
+  # with default "from" parameter.
7 8 
   config.mailer_sender = "please-change-me-at-config-initializers-devise@example.com"
8 9
9 10 
   # Configure the class responsible to send e-mails.
 
@@ -72,6 +73,12 @@ Devise.setup do |config|
72 73 
   # passing :skip => :sessions to `devise_for` in your config/routes.rb
73 74 
   config.skip_session_storage = [:http_auth]
74 75
76 
+  # By default, Devise cleans up the CSRF token on authentication to
77 
+  # avoid CSRF token fixation attacks. This means that, when using AJAX
78 
+  # requests for sign in and sign up, you need to get a new CSRF token
79 
+  # from the server. You can disable this option at your own risk.
80 
+  # config.clean_up_csrf_token_on_authentication = true
81 
+
75 82 
   # ==> Configuration for :database_authenticatable
76 83 
   # For bcrypt, this is the cost for hashing the password and defaults to 10. If
77 84 
   # using other encryptors, it sets how many times you want the password re-encrypted.
 
@@ -174,10 +181,6 @@ Devise.setup do |config|
174 181 
   # REST_AUTH_SITE_KEY to pepper)
175 182 
   # config.encryptor = :sha512
176 183
177 
-  # ==> Configuration for :token_authenticatable
178 
-  # Defines name of the authentication token params key
179 
-  # config.token_authentication_key = :auth_token
180 
-
181 184 
   # ==> Scopes configuration
182 185 
   # Turn scoped views on. Before rendering "sessions/new", it will first check for
183 186 
   # "users/sessions/new". It's turned off by default because it's slower if you

+ 1 - 1
config/initializers/secret_token.rb
View File 

@@ -4,4 +4,4 @@
4 4 
 # If you change this key, all old signed cookies will become invalid!
5 5 
 # Make sure the secret is at least 30 characters and all random,
6 6 
 # no regular words or you'll be exposed to dictionary attacks.
7 
-Huginn::Application.config.secret_token = ENV['APP_SECRET_TOKEN']
7 
+Huginn::Application.config.secret_key_base = ENV['APP_SECRET_TOKEN']

+ 4 - 4
config/routes.rb
View File 

@@ -29,18 +29,18 @@ Huginn::Application.routes.draw do
29 29
30 30 
   resources :user_credentials, :except => :show
31 31
32 
-  match "/worker_status" => "worker_status#show"
32 
+  get "/worker_status" => "worker_status#show"
33 33
34 34 
   post "/users/:user_id/update_location/:secret" => "user_location_updates#create"
35 35
36 
-  match "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests
36 
+  match  "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests, :via => [:get, :post, :put, :delete]
37 37 
   post "/users/:user_id/webhooks/:agent_id/:secret" => "web_requests#handle_request" # legacy
38 38
39 39 
 # To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
40 
-#  match "/delayed_job" => DelayedJobWeb, :anchor => false
40 
+#  get "/delayed_job" => DelayedJobWeb, :anchor => false
41 41
42 42 
   devise_for :users, :sign_out_via => [ :post, :delete ]
43 43
44 
-  match "/about" => "home#about"
44 
+  get "/about" => "home#about"
45 45 
   root :to => "home#index"
46 46 
 end

+ 1 - 1
db/seeds.rb
View File 

@@ -1,7 +1,7 @@
1 1 
 # This file should contain all the record creation needed to seed the database with its default values.
2 2 
 # The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
3 3
4 
-user = User.find_or_initialize_by_email("admin@example.com")
4 
+user = User.find_or_initialize_by(:email => "admin@example.com")
5 5 
 user.username = "admin"
6 6 
 user.password = "password"
7 7 
 user.password_confirmation = "password"

+ 3 - 0
deployment/.chef/knife.rb
View File 

@@ -0,0 +1,3 @@
1 
+cookbook_path ["cookbooks", "site-cookbooks"]
2 
+role_path     "roles"
3 
+data_bag_path "data_bags"

+ 71 - 0
deployment/Cheffile.lock
View File 

@@ -0,0 +1,71 @@
1 
+SITE
2 
+  remote: http://community.opscode.com/api/v1
3 
+  specs:
4 
+    apt (2.3.8)
5 
+    bluepill (2.3.1)
6 
+      rsyslog (>= 0.0.0)
7 
+    build-essential (2.0.0)
8 
+    chef_handler (1.1.6)
9 
+    dmg (2.2.0)
10 
+    ohai (1.1.12)
11 
+    rsyslog (1.12.2)
12 
+    runit (1.5.10)
13 
+      build-essential (>= 0.0.0)
14 
+      yum (~> 3.0)
15 
+      yum-epel (>= 0.0.0)
16 
+    windows (1.30.2)
17 
+      chef_handler (>= 0.0.0)
18 
+    yum (3.2.0)
19 
+    yum-epel (0.3.6)
20 
+      yum (~> 3.0)
21 
+
22 
+GIT
23 
+  remote: git://github.com/mdxp/nodejs-cookbook.git
24 
+  ref: master
25 
+  sha: e2415cd8c4e03dccf21d7ef6ca31e1c5c81467ca
26 
+  specs:
27 
+    nodejs (1.3.0)
28 
+      apt (>= 0.0.0)
29 
+      build-essential (>= 0.0.0)
30 
+      yum-epel (>= 0.0.0)
31 
+
32 
+GIT
33 
+  remote: git://github.com/opscode-cookbooks/git.git
34 
+  ref: master
35 
+  sha: 76b0f9bb08fdd9e2e201fd70b72298097accdf96
36 
+  specs:
37 
+    git (4.0.1)
38 
+      build-essential (>= 0.0.0)
39 
+      dmg (>= 0.0.0)
40 
+      runit (>= 1.0)
41 
+      windows (>= 0.0.0)
42 
+      yum (~> 3.0)
43 
+      yum-epel (>= 0.0.0)
44 
+
45 
+GIT
46 
+  remote: git://github.com/opscode-cookbooks/mysql.git
47 
+  ref: master
48 
+  sha: a2ff53f0ca6deca75aebf6da55ac381194ec7728
49 
+  specs:
50 
+    mysql (5.1.9)
51 
+
52 
+GIT
53 
+  remote: git://github.com/opscode-cookbooks/nginx.git
54 
+  ref: master
55 
+  sha: 05b3a613f53a0b05c96f9206c5d67aa420f337fb
56 
+  specs:
57 
+    nginx (2.6.3)
58 
+      apt (~> 2.2)
59 
+      bluepill (~> 2.3)
60 
+      build-essential (~> 2.0)
61 
+      ohai (~> 1.1)
62 
+      runit (~> 1.2)
63 
+      yum-epel (~> 0.3)
64 
+
65 
+DEPENDENCIES
66 
+  git (>= 0)
67 
+  mysql (>= 0)
68 
+  nginx (>= 0)
69 
+  nodejs (>= 0)
70 
+  runit (>= 0)
71 
+

+ 12 - 37
deployment/Vagrantfile
View File 

@@ -3,47 +3,22 @@
3 3
4 4 
 Vagrant.configure("2") do |config|
5 5 
   config.omnibus.chef_version = :latest
6 
-  config.vm.define :vb do |vb|
7 
-    vb.vm.box = "precise32"
8 
-    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
9 
-    vb.vm.network :forwarded_port, host: 3000, guest: 3000
10 6
11 
-    vb.vm.provision :chef_solo do |chef|
12 
-      chef.roles_path = "roles"
13 
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
14 
-      chef.add_role("huginn_development")
15 
-    end
7 
+  config.vm.provision :chef_solo do |chef|
8 
+    chef.roles_path = "roles"
9 
+    chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
10 
+    chef.add_role("huginn_development")
11 
+    # chef.add_role("huginn_production")
16 12 
   end
17 13
18 
-  config.vm.define :prl do |prl|
19 
-    prl.vm.box = "parallels/ubuntu-12.04"
20 
-
21 
-    prl.vm.provision :chef_solo do |chef|
22 
-      chef.roles_path = "roles"
23 
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
24 
-      chef.add_role("huginn_development")
25 
-    end
14 
+  config.vm.provider :virtualbox do |vb, override|
15 
+    #vb.memory = 1024
16 
+    #vb.cpus = 4
17 
+    override.vm.box = "hashicorp/precise64"
18 
+    override.vm.network :forwarded_port, host: 3000, guest: 3000
26 19 
   end
27 20
28 
-  config.vm.define :ec2 do |ec2|
29 
-    ec2.vm.box = "dummy"
30 
-    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
31 
-
32 
-    ec2.vm.provider :aws do |aws, override|
33 
-      aws.access_key_id = ""
34 
-      aws.secret_access_key = ""
35 
-      aws.keypair_name = ""
36 
-      aws.region = "us-east-1"
37 
-      aws.ami = "ami-d0f89fb9"
38 
-
39 
-      override.ssh.username = "ubuntu"
40 
-      override.ssh.private_key_path = ""
41 
-    end
42 
-    ec2.vm.provision :chef_solo do |chef|
43 
-      chef.roles_path = "roles"
44 
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
45 
-      chef.add_role("huginn_production")
46 
-
47 
-    end
21 
+  config.vm.provider :parallels do |prl, override|
22 
+    override.vm.box = "parallels/ubuntu-12.04"
48 23 
   end
49 24 
 end

+ 1 - 0
deployment/roles/huginn_development.json
View File 

@@ -23,6 +23,7 @@
23 23 
              "recipe[git]",
24 24 
              "recipe[apt]",
25 25 
              "recipe[mysql::server]",
26 
+             "recipe[mysql::client]",
26 27 
              "recipe[nodejs::install_from_binary]",
27 28 
              "recipe[huginn_development]"
28 29 
            ]

+ 1 - 1
deployment/roles/huginn_production.json
View File 

@@ -10,7 +10,7 @@
10 10
11 11 
 "default_attributes" : {
12 12 
   "mysql": {
13 
-    "server_root_password": "",
13 
+    "server_root_password": "password",
14 14 
     "server_repl_password": "",
15 15 
     "server_debian_password": ""
16 16 
   },

+ 1 - 1
deployment/site-cookbooks/huginn_development/recipes/default.rb
View File 

@@ -55,7 +55,7 @@ bash "huginn dependencies" do
55 55 
     export LANG="en_US.UTF-8"
56 56 
     export LC_ALL="en_US.UTF-8"
57 57 
     sudo bundle install
58 
-    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
58 
+    sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
59 59 
     sudo bundle exec rake db:create
60 60 
     sudo bundle exec rake db:migrate
61 61 
     sudo bundle exec rake db:seed

+ 0 - 58
deployment/site-cookbooks/huginn_production/files/default/Gemfile
View File 

@@ -1,58 +0,0 @@
1 
-source 'https://rubygems.org'
2 
-
3 
-gem 'rails'
4 
-gem 'rake'
5 
-gem 'mysql2'
6 
-gem 'devise'
7 
-gem 'kaminari'
8 
-gem 'bootstrap-kaminari-views'
9 
-gem "rufus-scheduler", :require => false
10 
-gem 'json', '>= 1.7.7'
11 
-gem 'jsonpath'
12 
-gem 'twilio-ruby'
13 
-
14 
-gem 'delayed_job', :git => 'https://github.com/wok/delayed_job' # Until the YAML issues are fixed in master.
15 
-gem 'delayed_job_active_record', "~> 0.3.3" # newer was giving a strange MySQL error
16 
-gem "daemons"
17 
-# gem "delayed_job_web"
18 
-group :production do
19 
-  gem 'unicorn'
20 
-end
21 
-gem 'foreman'
22 
-gem 'dotenv-rails', :groups => [:development, :test]
23 
-
24 
-group :assets do
25 
-  gem 'sass-rails',   '~> 3.2.3'
26 
-  gem 'coffee-rails', '~> 3.2.1'
27 
-  gem 'uglifier', '>= 1.0.3'
28 
-  gem 'select2-rails'
29 
-  gem 'jquery-rails'
30 
-end
31 
-
32 
-gem 'geokit-rails3'
33 
-gem 'kramdown'
34 
-gem "typhoeus"
35 
-gem 'nokogiri'
36 
-gem 'wunderground'
37 
-
38 
-gem "twitter"
39 
-gem 'twitter-stream', '>=0.1.16'
40 
-gem 'em-http-request'
41 
-
42 
-platforms :ruby_18 do
43 
-  gem 'system_timer'
44 
-  gem 'fastercsv'
45 
-end
46 
-
47 
-group :development do
48 
-  gem 'pry'
49 
-end
50 
-
51 
-group :development, :test do
52 
-  gem 'rspec-rails'
53 
-  gem 'rspec'
54 
-  gem 'shoulda-matchers'
55 
-  gem 'rr'
56 
-  gem 'webmock', :require => false
57 
-  gem 'rake'
58 
-end

+ 4 - 4
deployment/site-cookbooks/huginn_production/files/default/Procfile
View File 

@@ -1,4 +1,4 @@
1 
-web: sudo bundle exec unicorn_rails -c config/unicorn.rb
2 
-schedule: sudo bundle exec rails runner bin/schedule.rb
3 
-twitter: sudo bundle exec rails runner bin/twitter_stream.rb
4 
-dj: sudo bundle exec script/delayed_job run
1 
+web: sudo bundle exec unicorn_rails -c config/unicorn.rb -E production
2 
+schedule: sudo RAILS_ENV=production bundle exec rails runner bin/schedule.rb
3 
+twitter: sudo RAILS_ENV=production bundle exec rails runner bin/twitter_stream.rb
4 
+dj: sudo RAILS_ENV=production bundle exec script/delayed_job run

+ 2 - 1
deployment/site-cookbooks/huginn_production/files/default/env.example
View File 

@@ -14,7 +14,7 @@ DATABASE_RECONNECT=true
14 14 
 DATABASE_NAME=huginn_production
15 15 
 DATABASE_POOL=5
16 16 
 DATABASE_USERNAME=root
17 
-DATABASE_PASSWORD=
17 
+DATABASE_PASSWORD=password
18 18 
 #DATABASE_HOST=your-domain-here.com
19 19 
 #DATABASE_PORT=3306
20 20 
 #DATABASE_SOCKET=/tmp/mysql.sock
 
@@ -23,6 +23,7 @@ DATABASE_PASSWORD=
23 23
24 24 
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
25 25 
 RAILS_ENV=production
26 
+FORCE_SSL=false
26 27
27 28 
 # Outgoing email settings.  To use Gmail or Google Apps, put your Google Apps domain or gmail.com
28 29 
 # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.

+ 5 - 6
deployment/site-cookbooks/huginn_production/files/default/nginx.conf
View File 

@@ -1,15 +1,18 @@
1 1 
 #worker_process 2;
2 2 
 user huginn huginn;
3 3
4 
-events {
4 
+events {
5 5 
   worker_connections 1024;
6 6 
   accept_mutex on;
7 7 
 }
8 8
9 9 
 http {
10 
+  types_hash_max_size 2048;
11 
+  include    mime.types;
12 
+
10 13 
   upstream huginn_server {
11 14 
     server unix:/home/huginn/shared/tmp/sockets/unicorn.sock;
12 
-}
15 
+  }
13 16
14 17 
   server {
15 18 
     listen 80;
 
@@ -23,13 +26,9 @@ http {
23 26 
     }
24 27 
     location @app {
25 28 
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
26 
-
27 29 
       proxy_set_header X-Forwarded-Proto $scheme;
28 
-
29 30 
       proxy_set_header Host $http_host;
30 
-
31 31 
       proxy_redirect off;
32 
-
33 32 
       proxy_pass http://huginn_server;
34 33 
     }
35 34 
 }

+ 4 - 2
deployment/site-cookbooks/huginn_production/files/default/unicorn.rb
View File 

@@ -17,7 +17,8 @@ stdout_path "log/unicorn_err.log"
17 17 
 pid '/home/huginn/shared/tmp/pids/unicorn.pid'
18 18
19 19 
 before_fork do |server, worker|
20 
-  ActiveRecord::Base.connection.disconnect!
20 
+  defined?(ActiveRecord::Base) and
21 
+    ActiveRecord::Base.connection.disconnect!
21 22 
   old_pid = "#{server.config[:pid]}.oldbin"
22 23 
   if File.exists?(old_pid) && server.pid != old_pid
23 24 
     begin
 
@@ -29,5 +30,6 @@ before_fork do |server, worker|
29 30 
 end
30 31
31 32 
 after_fork do |server, worker|
32 
-  ActiveRecord::Base.establish_connection
33 
+  defined?(ActiveRecord::Base) and
34 
+    ActiveRecord::Base.establish_connection
33 35 
 end

+ 18 - 9
deployment/site-cookbooks/huginn_production/recipes/default.rb
View File 

@@ -14,10 +14,17 @@ group "huginn" do
14 14 
   members ["huginn"]
15 15 
 end
16 16
17 
-%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8" "libmysqlclient-dev").each do |pkg|
17 
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8" "libmysqlclient-dev" "libffi-dev" "libssl-dev" "rubygems").each do |pkg|
18 18 
   package("#{pkg}")
19 19 
 end
20 20
21 
+bash "Setting default ruby version to 1.9" do
22 
+  code <<-EOH
23 
+    update-alternatives --set ruby /usr/bin/ruby1.9.1
24 
+    update-alternatives --set gem /usr/bin/gem1.9.1
25 
+  EOH
26 
+end
27 
+
21 28 
 gem_package("rake")
22 29 
 gem_package("bundle")
23 30 
 
@@ -36,6 +43,7 @@ end
36 43
37 44 
 deploy "/home/huginn" do
38 45 
   repo "https://github.com/cantino/huginn.git"
46 
+  branch "master"
39 47 
   user "huginn"
40 48 
   group "huginn"
41 49 
   environment "RAILS_ENV" => "production"
 
@@ -56,7 +64,7 @@ deploy "/home/huginn" do
56 64 
     end
57 65 
     directory("/home/huginn/shared/tmp/pids")
58 66 
     directory("/home/huginn/shared/tmp/sockets")
59 
-    %w(Procfile unicorn.rb Gemfile nginx.conf).each do |file|
67 
+    %w(Procfile unicorn.rb nginx.conf).each do |file|
60 68 
       cookbook_file "/home/huginn/shared/config/#{file}" do
61 69 
       owner "huginn"
62 70 
       action :create_if_missing
 
@@ -77,16 +85,17 @@ deploy "/home/huginn" do
77 85 
       code <<-EOH
78 86 
       export LANG="en_US.UTF-8"
79 87 
       export LC_ALL="en_US.UTF-8"
80 
-      ln -nfs /home/huginn/shared/config/Gemfile ./Gemfile
81 88 
       ln -nfs /home/huginn/shared/config/Procfile ./Procfile
82 89 
       ln -nfs /home/huginn/shared/config/.env ./.env
83 90 
       ln -nfs /home/huginn/shared/config/unicorn.rb ./config/unicorn.rb
84 
-      sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/
85 
-      sudo bundle install
86 
-      sed -i s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env
87 
-      sudo bundle exec rake db:create
88 
-      sudo bundle exec rake db:migrate
89 
-      sudo bundle exec rake db:seed
91 
+      sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/
92 
+      echo 'gem "unicorn", :group => :production' >> Gemfile
93 
+      sudo bundle install --without=development --without=test
94 
+      sed -i s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env
95 
+      sudo RAILS_ENV=production bundle exec rake db:create
96 
+      sudo RAILS_ENV=production bundle exec rake db:migrate
97 
+      sudo RAILS_ENV=production bundle exec rake db:seed
98 
+      sudo RAILS_ENV=production bundle exec rake assets:precompile
90 99 
       sudo foreman export upstart /etc/init -a huginn -u huginn -l log
91 100 
       sudo start huginn
92 101 
       EOH

+ 0 - 6
deployment/solo.rb
View File 

@@ -1,6 +0,0 @@
1 
-file_cache_path           "/tmp/chef-solo"
2 
-data_bag_path             "/tmp/chef-solo/data_bags"
3 
-encrypted_data_bag_secret "/tmp/chef-solo/data_bag_key"
4 
-cookbook_path             [ "/tmp/chef-solo/site-cookbooks",
5 
-                            "/tmp/chef-solo/cookbooks" ]
6 
-role_path                 "/tmp/chef-solo/roles"

+ 2 - 2
lib/rdbms_functions.rb
View File 

@@ -1,10 +1,10 @@
1 1 
 module RDBMSFunctions
2 2 
   def rdbms_date_add(source, unit, amount)
3 
-    adapter_type = connection.adapter_name.downcase.to_sym
3 
+    adapter_type = ActiveRecord::Base.connection.adapter_name.downcase.to_sym
4 4 
     case adapter_type
5 5 
       when :mysql, :mysql2
6 6 
         "DATE_ADD(`#{source}`, INTERVAL #{amount} #{unit})"
7 
-      when :postgresql
7 
+      when :postgresql
8 8 
         "(#{source} + INTERVAL '#{amount} #{unit}')"
9 9 
       else
10 10 
         raise NotImplementedError, "Unknown adapter type '#{adapter_type}'"

+ 16 - 0
spec/controllers/agents_controller_spec.rb
View File 

@@ -46,6 +46,22 @@ describe AgentsController do
46 46 
     end
47 47 
   end
48 48
49 
+  describe "GET new with :id" do
50 
+    it "opens a clone of a given Agent" do
51 
+      sign_in users(:bob)
52 
+      get :new, :id => agents(:bob_website_agent).to_param
53 
+      assigns(:agent).attributes.should eq(users(:bob).agents.build_clone(agents(:bob_website_agent)).attributes)
54 
+    end
55 
+
56 
+    it "only allows the current user to clone his own Agent" do
57 
+      sign_in users(:bob)
58 
+
59 
+      lambda {
60 
+        get :new, :id => agents(:jane_website_agent).to_param
61 
+      }.should raise_error(ActiveRecord::RecordNotFound)
62 
+    end
63 
+  end
64 
+
49 65 
   describe "GET edit" do
50 66 
     it "only shows Agents for the current user" do
51 67 
       sign_in users(:bob)

+ 17 - 0
spec/data_fixtures/stubhub_data.json
View File 

@@ -0,0 +1,17 @@
1 
+{
2 
+  "response":{
3 
+    "docs":[
4 
+      {
5 
+      "url": "http://www.stubhub.com/event/name-1-1-2014-12345",
6 
+      "seo_description_en_US": "name",
7 
+      "event_date_local": "2014-01-01",
8 
+      "maxPrice": "100",
9 
+      "minPrice": "50",
10 
+      "totalPostings": "100",
11 
+      "totalTickets": "200",
12 
+      "venue_name": "Venue Name"
13 
+    }
14 
+    ]
15 
+  }
16 
+}
17 
+

+ 48 - 0
spec/helpers/dot_helper_spec.rb
View File 

@@ -0,0 +1,48 @@
1 
+require 'spec_helper'
2 
+
3 
+describe DotHelper do
4 
+  describe "#dot_id" do
5 
+    it "properly escapes double quotaion and backslash" do
6 
+      dot_id('hello\\"').should == '"hello\\\\\\""'
7 
+    end
8 
+  end
9 
+
10 
+  describe "with example Agents" do
11 
+    class Agents::DotFoo < Agent
12 
+      default_schedule "2pm"
13 
+
14 
+      def check
15 
+        create_event :payload => {}
16 
+      end
17 
+    end
18 
+
19 
+    class Agents::DotBar < Agent
20 
+      cannot_be_scheduled!
21 
+
22 
+      def check
23 
+        create_event :payload => {}
24 
+      end
25 
+    end
26 
+
27 
+    before do
28 
+      stub(Agents::DotFoo).valid_type?("Agents::DotFoo") { true }
29 
+      stub(Agents::DotBar).valid_type?("Agents::DotBar") { true }
30 
+    end
31 
+
32 
+    describe "#agents_dot" do
33 
+      it "generates a DOT script" do
34 
+        @foo = Agents::DotFoo.new(:name => "foo")
35 
+        @foo.user = users(:bob)
36 
+        @foo.save!
37 
+
38 
+        @bar = Agents::DotBar.new(:name => "bar")
39 
+        @bar.user = users(:bob)
40 
+        @bar.sources << @foo
41 
+        @bar.save!
42 
+
43 
+        agents_dot([@foo, @bar]).should == 'digraph foo {"foo";"foo"->"bar";"bar";}'
44 
+        agents_dot([@foo, @bar], true).should == 'digraph foo {"foo"[URL="/agents/%d"];"foo"->"bar";"bar"[URL="/agents/%d"];}' % [@foo.id, @bar.id]
45 
+      end
46 
+    end
47 
+  end
48 
+end

+ 1 - 1
spec/lib/utils_spec.rb
View File 

@@ -97,7 +97,7 @@ describe Utils do
97 97 
     it "escapes </script> tags in the output JSON" do
98 98 
       cleaned_json = Utils.jsonify(:foo => "bar", :xss => "</script><script>alert('oh no!')</script>")
99 99 
       cleaned_json.should_not include("</script>")
100 
-      cleaned_json.should include("<\\/script>")
100 
+      cleaned_json.should include('\\u003c/script\\u003e')
101 101 
     end
102 102
103 103 
     it "html_safes the output unless :skip_safe is passed in" do

+ 45 - 0
spec/models/agent_spec.rb
View File 

@@ -514,6 +514,51 @@ describe Agent do
514 514 
         end
515 515 
       end
516 516 
     end
517 
+
518 
+    describe "Agent.build_clone" do
519 
+      before do
520 
+        Event.delete_all
521 
+        @sender = Agents::SomethingSource.new(
522 
+          name: 'Agent (2)',
523 
+          options: { foo: 'bar2' },
524 
+          schedule: '5pm')
525 
+        @sender.user = users(:bob)
526 
+        @sender.save!
527 
+        @sender.create_event :payload => {}
528 
+        @sender.create_event :payload => {}
529 
+        @sender.events.count.should == 2
530 
+
531 
+        @receiver = Agents::CannotBeScheduled.new(
532 
+          name: 'Agent',
533 
+          options: { foo: 'bar3' },
534 
+          keep_events_for: 3,
535 
+          propagate_immediately: true)
536 
+        @receiver.user = users(:bob)
537 
+        @receiver.sources << @sender
538 
+        @receiver.memory[:test] = 1
539 
+        @receiver.save!
540 
+      end
541 
+
542 
+      it "should create a clone of a given agent for editing" do
543 
+        sender_clone = users(:bob).agents.build_clone(@sender)
544 
+
545 
+        sender_clone.attributes.should == Agent.new.attributes.
546 
+          update(@sender.slice(:user_id, :type,
547 
+            :options, :schedule, :keep_events_for, :propagate_immediately)).
548 
+          update('name' => 'Agent (2) (2)', 'options' => { 'foo' => 'bar2' })
549 
+
550 
+        sender_clone.source_ids.should == []
551 
+
552 
+        receiver_clone = users(:bob).agents.build_clone(@receiver)
553 
+
554 
+        receiver_clone.attributes.should == Agent.new.attributes.
555 
+          update(@receiver.slice(:user_id, :type,
556 
+            :options, :schedule, :keep_events_for, :propagate_immediately)).
557 
+          update('name' => 'Agent (3)', 'options' => { 'foo' => 'bar3' })
558 
+
559 
+        receiver_clone.source_ids.should == [@sender.id]
560 
+      end
561 
+    end
517 562 
   end
518 563
519 564 
   describe ".trigger_web_request" do

+ 1 - 1
spec/models/agents/hipchat_agent_spec.rb
View File 

@@ -62,7 +62,7 @@ describe Agents::HipchatAgent do
62 62 
     end
63 63
64 64 
     it "should merge all options" do
65 
-      @checker.send(:merge_options, @event).should == {
65 
+      @checker.send(:merge_options, @event).deep_symbolize_keys.should == {
66 66 
         :room_name => "test",
67 67 
         :username => "Huggin user",
68 68 
         :message => "Looks like its going to rain",

+ 19 - 14
spec/models/agents/public_transport_agent_spec.rb
View File 

@@ -19,7 +19,6 @@ describe Agents::PublicTransportAgent do
19 19 
       stub_request(:get, "http://webservices.nextbus.com/service/publicXMLFeed?a=sf-muni&command=predictionsForMultiStops&stops=N%7C5215").
20 20 
          with(:headers => {'User-Agent'=>'Typhoeus - https://github.com/typhoeus/typhoeus'}).
21 21 
          to_return(:status => 200, :body => File.read(Rails.root.join("spec/data_fixtures/public_transport_agent.xml")), :headers => {})
22 
-      stub(Time).now {"2014-01-14 20:21:30 +0500".to_time}
23 22 
     end
24 23
25 24 
     it "should create 4 events" do
 
@@ -27,15 +26,18 @@ describe Agents::PublicTransportAgent do
27 26 
     end
28 27
29 28 
     it "should add 4 items to memory" do
30 
-      @agent.memory.should == {}
31 
-      @agent.check
32 
-      @agent.memory.should == {"existing_routes" => [
33 
-          {"stopTag"=>"5221", "tripTag"=>"5840324", "epochTime"=>"1389706393991", "currentTime"=>"2014-01-14 20:21:30 +0500"},
34 
-          {"stopTag"=>"5221", "tripTag"=>"5840083", "epochTime"=>"1389706512784", "currentTime"=>"2014-01-14 20:21:30 +0500"},
35 
-          {"stopTag"=>"5215", "tripTag"=>"5840324", "epochTime"=>"1389706282012", "currentTime"=>"2014-01-14 20:21:30 +0500"},
36 
-          {"stopTag"=>"5215", "tripTag"=>"5840083", "epochTime"=>"1389706400805", "currentTime"=>"2014-01-14 20:21:30 +0500"}
37 
-        ]
38 
-      }
29 
+      time_travel_to Time.parse("2014-01-14 20:21:30 +0500") do
30 
+        @agent.memory.should == {}
31 
+        @agent.check
32 
+        @agent.save
33 
+        @agent.reload.memory.should == {"existing_routes" => [
34 
+            {"stopTag"=>"5221", "tripTag"=>"5840324", "epochTime"=>"1389706393991", "currentTime"=>Time.now.to_s},
35 
+            {"stopTag"=>"5221", "tripTag"=>"5840083", "epochTime"=>"1389706512784", "currentTime"=>Time.now.to_s},
36 
+            {"stopTag"=>"5215", "tripTag"=>"5840324", "epochTime"=>"1389706282012", "currentTime"=>Time.now.to_s},
37 
+            {"stopTag"=>"5215", "tripTag"=>"5840083", "epochTime"=>"1389706400805", "currentTime"=>Time.now.to_s}
38 
+          ]
39 
+        }
40 
+      end
39 41 
     end
40 42
41 43 
     it "should not create events twice" do
 
@@ -44,10 +46,13 @@ describe Agents::PublicTransportAgent do
44 46 
     end
45 47
46 48 
     it "should reset memory after 2 hours" do
47 
-      lambda { @agent.check }.should change {@agent.events.count}.by(4)
48 
-      stub(Time).now {"2014-01-14 20:21:30 +0500".to_time + 3.hours}
49 
-      @agent.cleanup_old_memory
50 
-      lambda { @agent.check }.should change {@agent.events.count}.by(4)
49 
+      time_travel_to Time.parse("2014-01-14 20:21:30 +0500") do
50 
+        lambda { @agent.check }.should change {@agent.events.count}.by(4)
51 
+      end
52 
+      time_travel_to "2014-01-14 23:21:30 +0500".to_time do
53 
+        @agent.cleanup_old_memory
54 
+        lambda { @agent.check }.should change {@agent.events.count}.by(4)
55 
+      end
51 56 
     end
52 57 
   end
53 58

+ 67 - 0
spec/models/agents/stubhub_agent_spec.rb
View File 

@@ -0,0 +1,67 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::StubhubAgent do
4 
+
5 
+  let(:name) { 'Agent Name' }
6 
+  let(:url) { 'http://www.stubhub.com/event/name-1-1-2014-12345' }
7 
+  let(:parsed_body) { JSON.parse(body)['response']['docs'][0] }
8 
+  let(:valid_params) { { 'url' => parsed_body['url'] } }
9 
+  let(:body) { File.read(Rails.root.join('spec/data_fixtures/stubhub_data.json')) }
10 
+  let(:stubhub_event_id) { 12345 }
11 
+  let(:response_payload) { {
12 
+                            'url' => url,
13 
+                            'name' => parsed_body['seo_description_en_US'],
14 
+                            'date' => parsed_body['event_date_local'],
15 
+                            'max_price' => parsed_body['maxPrice'],
16 
+                            'min_price' => parsed_body['minPrice'],
17 
+                            'total_postings' => parsed_body['totalPostings'],
18 
+                            'total_tickets' => parsed_body['totalTickets'],
19 
+                            'venue_name' => parsed_body['venue_name']
20 
+                            } }
21 
+
22 
+  before do
23 
+      stub_request(:get, "http://www.stubhub.com/listingCatalog/select/?q=%2B%20stubhubDocumentType:event%0D%0A%2B%20event_id:#{stubhub_event_id}%0D%0A&rows=10&start=0&wt=json").
24 
+         to_return(:status => 200, :body => body, :headers => {})
25 
+
26 
+    @stubhub_agent = described_class.new(name: name, options: valid_params)
27 
+    @stubhub_agent.user = users(:jane)
28 
+    @stubhub_agent.save!
29 
+  end
30 
+
31 
+
32 
+  describe "#check" do
33 
+
34 
+    it 'should create an event' do
35 
+      expect { @stubhub_agent.check }.to change { Event.count }.by(1)
36 
+    end
37 
+
38 
+    it 'should properly parse the response' do
39 
+      event = @stubhub_agent.check
40 
+      event.payload.should == response_payload
41 
+    end
42 
+  end
43 
+
44 
+  describe "validations" do
45 
+    before do
46 
+      @stubhub_agent.should be_valid
47 
+    end
48 
+
49 
+    it "should require a url" do
50 
+      @stubhub_agent.options['url'] = nil
51 
+      @stubhub_agent.should_not be_valid
52 
+    end
53 
+
54 
+  end
55 
+
56 
+  describe "#working?" do
57 
+    it "checks if events have been received within the expected receive period" do
58 
+      @stubhub_agent.should_not be_working
59 
+
60 
+      Agents::StubhubAgent.async_check @stubhub_agent.id
61 
+      @stubhub_agent.reload.should be_working
62 
+      two_days_from_now = 2.days.from_now
63 
+      stub(Time).now { two_days_from_now }
64 
+      @stubhub_agent.reload.should_not be_working
65 
+    end
66 
+  end
67 
+end

+ 57 - 1
spec/models/agents/trigger_agent_spec.rb
View File 

@@ -30,9 +30,32 @@ describe Agents::TriggerAgent do
30 30 
       @checker.should be_valid
31 31 
     end
32 32
33 
-    it "should validate presence of options" do
33 
+    it "should validate presence of message" do
34 34 
       @checker.options['message'] = nil
35 35 
       @checker.should_not be_valid
36 
+
37 
+      @checker.options['message'] = ''
38 
+      @checker.should_not be_valid
39 
+    end
40 
+
41 
+    it "should be valid without a message when 'keep_event' is set" do
42 
+      @checker.options['keep_event'] = 'true'
43 
+      @checker.options['message'] = ''
44 
+      @checker.should be_valid
45 
+    end
46 
+
47 
+    it "if present, 'keep_event' must equal true or false" do
48 
+      @checker.options['keep_event'] = 'true'
49 
+      @checker.should be_valid
50 
+
51 
+      @checker.options['keep_event'] = 'false'
52 
+      @checker.should be_valid
53 
+
54 
+      @checker.options['keep_event'] = ''
55 
+      @checker.should be_valid
56 
+
57 
+      @checker.options['keep_event'] = 'tralse'
58 
+      @checker.should_not be_valid
36 59 
     end
37 60
38 61 
     it "should validate the three fields in each rule" do
 
@@ -278,5 +301,38 @@ describe Agents::TriggerAgent do
278 301 
         @checker.receive([@event])
279 302 
       }.should_not change { Event.count }
280 303 
     end
304 
+
305 
+    describe "when 'keep_event' is true" do
306 
+      before do
307 
+        @checker.options['keep_event'] = 'true'
308 
+        @event.payload['foo']['bar']['baz'] = "5"
309 
+        @checker.options['rules'].first['type'] = "field<value"
310 
+      end
311 
+
312 
+      it "can re-emit the origin event" do
313 
+        @checker.options['rules'].first['value'] = 3
314 
+        @checker.options['message'] = ''
315 
+        @event.payload['message'] = 'hi there'
316 
+
317 
+        lambda {
318 
+          @checker.receive([@event])
319 
+        }.should_not change { Event.count }
320 
+
321 
+        @checker.options['rules'].first['value'] = 6
322 
+        lambda {
323 
+          @checker.receive([@event])
324 
+        }.should change { Event.count }.by(1)
325 
+
326 
+        @checker.most_recent_event.payload.should == @event.payload
327 
+      end
328 
+
329 
+      it "merges 'message' into the original event when present" do
330 
+        @checker.options['rules'].first['value'] = 6
331 
+
332 
+        @checker.receive([@event])
333 
+
334 
+        @checker.most_recent_event.payload.should == @event.payload.merge(:message => "I saw '5' from Joe")
335 
+      end
336 
+    end
281 337 
   end
282 338 
 end

+ 97 - 11
spec/models/agents/website_agent_spec.rb
View File 

@@ -21,28 +21,71 @@ describe Agents::WebsiteAgent do
21 21 
       @checker.save!
22 22 
     end
23 23
24 
-    describe "#check" do
24 
+    describe "validations" do
25 
+      before do
26 
+        @checker.should be_valid
27 
+      end
28 
+
25 29 
       it "should validate the integer fields" do
26 
-        @checker.options['expected_update_period_in_days'] = "nonsense"
27 
-        lambda { @checker.save! }.should raise_error;
28 30 
         @checker.options['expected_update_period_in_days'] = "2"
31 
+        @checker.should be_valid
32 
+
33 
+        @checker.options['expected_update_period_in_days'] = "nonsense"
34 
+        @checker.should_not be_valid
35 
+      end
36 
+
37 
+      it "should validate uniqueness_look_back" do
29 38 
         @checker.options['uniqueness_look_back'] = "nonsense"
30 
-        lambda { @checker.save! }.should raise_error;
39 
+        @checker.should_not be_valid
40 
+
41 
+        @checker.options['uniqueness_look_back'] = "2"
42 
+        @checker.should be_valid
43 
+      end
44 
+
45 
+      it "should validate headers" do
46 
+        @checker.options['headers'] = "blah"
47 
+        @checker.should_not be_valid
48 
+
49 
+        @checker.options['headers'] = ""
50 
+        @checker.should be_valid
51 
+
52 
+        @checker.options['headers'] = {}
53 
+        @checker.should be_valid
54 
+
55 
+        @checker.options['headers'] = { 'foo' => 'bar' }
56 
+        @checker.should be_valid
57 
+      end
58 
+
59 
+      it "should validate mode" do
31 60 
         @checker.options['mode'] = "nonsense"
32 
-        lambda { @checker.save! }.should raise_error;
33 
-        @checker.options = @site
61 
+        @checker.should_not be_valid
62 
+
63 
+        @checker.options['mode'] = "on_change"
64 
+        @checker.should be_valid
65 
+
66 
+        @checker.options['mode'] = "all"
67 
+        @checker.should be_valid
68 
+
69 
+        @checker.options['mode'] = ""
70 
+        @checker.should be_valid
34 71 
       end
35 72
36 73 
       it "should validate the force_encoding option" do
74 
+        @checker.options['force_encoding'] = ''
75 
+        @checker.should be_valid
76 
+
37 77 
         @checker.options['force_encoding'] = 'UTF-8'
38 
-        lambda { @checker.save! }.should_not raise_error;
78 
+        @checker.should be_valid
79 
+
39 80 
         @checker.options['force_encoding'] = ['UTF-8']
40 
-        lambda { @checker.save! }.should raise_error;
81 
+        @checker.should_not be_valid
82 
+
41 83 
         @checker.options['force_encoding'] = 'UTF-42'
42 
-        lambda { @checker.save! }.should raise_error;
43 
-        @checker.options = @site
84 
+        @checker.should_not be_valid
44 85 
       end
86 
+    end
45 87
88 
+    describe "#check" do
46 89 
       it "should check for changes (and update Event.expires_at)" do
47 90 
         lambda { @checker.check }.should change { Event.count }.by(1)
48 91 
         event = Event.last
 
@@ -331,11 +374,26 @@ describe Agents::WebsiteAgent do
331 374 
         end
332 375 
       end
333 376 
     end
377 
+
378 
+    describe "#receive" do
379 
+      it "should scrape from the url element in incoming event payload" do
380 
+        @event = Event.new
381 
+        @event.agent = agents(:bob_rain_notifier_agent)
382 
+        @event.payload = { 'url' => "http://xkcd.com" }
383 
+
384 
+        lambda {
385 
+          @checker.options = @site
386 
+          @checker.receive([@event])
387 
+        }.should change { Event.count }.by(1)
388 
+      end
389 
+    end
334 390 
   end
335 391
336 392 
   describe "checking with http basic auth" do
337 393 
     before do
338 
-      stub_request(:any, /user:pass/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
394 
+      stub_request(:any, /example/).
395 
+        with(headers: { 'Authorization' => "Basic #{['user:pass'].pack('m').chomp}" }).
396 
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
339 397 
       @site = {
340 398 
         'name' => "XKCD",
341 399 
         'expected_update_period_in_days' => 2,
 
@@ -361,4 +419,32 @@ describe Agents::WebsiteAgent do
361 419 
       end
362 420 
     end
363 421 
   end
422 
+
423 
+  describe "checking with headers" do
424 
+    before do
425 
+      stub_request(:any, /example/).
426 
+        with(headers: { 'foo' => 'bar', 'user_agent' => /Faraday/ }).
427 
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
428 
+      @site = {
429 
+        'name' => "XKCD",
430 
+        'expected_update_period_in_days' => 2,
431 
+        'type' => "html",
432 
+        'url' => "http://www.example.com",
433 
+        'mode' => 'on_change',
434 
+        'headers' => { 'foo' => 'bar' },
435 
+        'extract' => {
436 
+          'url' => { 'css' => "#comic img", 'attr' => "src" },
437 
+        }
438 
+      }
439 
+      @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @site)
440 
+      @checker.user = users(:bob)
441 
+      @checker.save!
442 
+    end
443 
+
444 
+    describe "#check" do
445 
+      it "should check for changes" do
446 
+        lambda { @checker.check }.should change { Event.count }.by(1)
447 
+      end
448 
+    end
449 
+  end
364 450 
 end

+ 1 - 0
spec/spec_helper.rb
View File 

@@ -42,4 +42,5 @@ RSpec.configure do |config|
42 42
43 43 
   config.include Devise::TestHelpers, :type => :controller
44 44 
   config.include SpecHelpers
45 
+  config.include Delorean
45 46 
 end